<commit_before>from multiprocessing import Process, Queue
from urllib.parse import urlparse
import requests
import pandas as pd
import sqlalchemy as s
from sqlalchemy.ext.automap import automap_base
from sqlalchemy import MetaData
import statistics
import logging
import json
import numpy as np
import scipy.stats
logging.basicConfig(filename='worker.log', filemode='w', level=logging.INFO)

def dump_queue(queue):
    """
    Empties all pending items in a queue and returns them in a list.
    """
    result = []
    queue.put("STOP")
    for i in iter(queue.get, 'STOP'):
        result.append(i)
    # time.sleep(.1)
    return result

class InsightWorker:
    """ Worker that collects data from the Github API and stores it in our database
    task: most recent task the broker added to the worker's queue
    child: current process of the queue being ran
    queue: queue of tasks to be fulfilled
    config: holds info like api keys, descriptions, and database connection strings
    """
    def __init__(self, config, task=None):
        self._task = task
        self._child = None
        self._queue = Queue()
        self.config = config
        self.db = None
        self.tool_source = 'Insight Worker'
        self.tool_version = '0.0.1' # See __init__.py
        self.data_source = 'Augur API'

        logging.info("Worker initializing...")
        
        specs = {
            "id": "com.augurlabs.core.insight_worker",
            "location": self.config['location'],
            "qualifications":  [
                {
                    "given": [["repo_git"]],
                    "models":["insights"]
                }
            ],
            "config": [self.config]
        }

        self.metric_results_counter = 0
        self.insight_results_counter = 0

        """
        Connect to GHTorrent
        
        :param dbstr: The [database string](http://docs.sqlalchemy.org/en/latest/core/engines.html) to connect to the GHTorrent database
        """
        self.DB_STR = 'postgresql://{}:{}@{}:{}/{}'.format(
            self.config['user'], self.config['password'], self.config['host'], self.config['port'], self.config['database']
        )

        dbschema='augur_data' # Searches left-to-right
        self.db = s.create_engine(self.DB_STR, poolclass=s.pool.NullPool,
            connect_args={'options': '-csearch_path={}'.format(dbschema)})

        
        # produce our own MetaData object
        metadata = MetaData()

        # we can reflect it ourselves from a database, using options
        # such as 'only' to limit what tables we look at...
        metadata.reflect(self.db, only=['chaoss_metric_status', 'repo_insights'])

        # we can then produce a set of mappings from this MetaData.
        Base = automap_base(metadata=metadata)

        # calling prepare() just sets up mapped classes and relationships.
        Base.prepare()

        # mapped classes are ready
        self.chaoss_metric_status_table = Base.classes['chaoss_metric_status'].__table__
        self.repo_insights_table = Base.classes['repo_insights'].__table__

        requests.post('http://localhost:{}/api/unstable/workers'.format(
            self.config['broker_port']), json=specs) #hello message

        # Query all repos and last repo id
        repoUrlSQL = s.sql.text("""
                SELECT repo_git, repo_id FROM repo ORDER BY repo_id DESC
            """)
        rs = pd.read_sql(repoUrlSQL, self.db, params={}).to_records()
        pop_off = 100
        i = 0
        while i < pop_off:
            rs = rs[1:]
            i += 1
        for row in rs:
            self._queue.put({'repo_id': row['repo_id'], 'repo_git': row['repo_git']})
        self.run()
        # self.discover_insights({'repo_id': 21420, 'repo_git': 'https://github.com/rails/ruby-coffee-script.git'})

    def update_config(self, config):
        """ Method to update config and set a default<commit_msg>insight_worker progress<commit_after>from multiprocessing import Process, Queue
from urllib.parse import urlparse
import requests
import pandas as pd
import sqlalchemy as s
from sqlalchemy.ext.automap import automap_base
from sqlalchemy import MetaData
import statistics
import logging
import json
import numpy as np
import scipy.stats
logging.basicConfig(filename='worker.log', filemode='w', level=logging.INFO)

def dump_queue(queue):
    """
    Empties all pending items in a queue and returns them in a list.
    """
    result = []
    queue.put("STOP")
    for i in iter(queue.get, 'STOP'):
        result.append(i)
    # time.sleep(.1)
    return result

class InsightWorker:
    """ Worker that collects data from the Github API and stores it in our database
    task: most recent task the broker added to the worker's queue
    child: current process of the queue being ran
    queue: queue of tasks to be fulfilled
    config: holds info like api keys, descriptions, and database connection strings
    """
    def __init__(self, config, task=None):
        self._task = task
        self._child = None
        self._queue = Queue()
        self.config = config
        self.db = None
        self.tool_source = 'Insight Worker'
        self.tool_version = '0.0.1' # See __init__.py
        self.data_source = 'Augur API'

        logging.info("Worker initializing...")
        
        specs = {
            "id": "com.augurlabs.core.insight_worker",
            "location": self.config['location'],
            "qualifications":  [
                {
                    "given": [["repo_git"]],
                    "models":["insights"]
                }
            ],
            "config": [self.config]
        }

        self.metric_results_counter = 0
        self.insight_results_counter = 0

        """
        Connect to GHTorrent
        
        :param dbstr: The [database string](http://docs.sqlalchemy.org/en/latest/core/engines.html) to connect to the GHTorrent database
        """
        self.DB_STR = 'postgresql://{}:{}@{}:{}/{}'.format(
            self.config['user'], self.config['password'], self.config['host'], self.config['port'], self.config['database']
        )

        dbschema='augur_data' # Searches left-to-right
        self.db = s.create_engine(self.DB_STR, poolclass=s.pool.NullPool,
            connect_args={'options': '-csearch_path={}'.format(dbschema)})

        
        # produce our own MetaData object
        metadata = MetaData()

        # we can reflect it ourselves from a database, using options
        # such as 'only' to limit what tables we look at...
        metadata.reflect(self.db, only=['chaoss_metric_status', 'repo_insights'])

        # we can then produce a set of mappings from this MetaData.
        Base = automap_base(metadata=metadata)

        # calling prepare() just sets up mapped classes and relationships.
        Base.prepare()

        # mapped classes are ready
        self.chaoss_metric_status_table = Base.classes['chaoss_metric_status'].__table__
        self.repo_insights_table = Base.classes['repo_insights'].__table__

        requests.post('http://localhost:{}/api/unstable/workers'.format(
            self.config['broker_port']), json=specs) #hello message

        # Query all repos and last repo id
        repoUrlSQL = s.sql.text("""
                SELECT repo_git, repo_id FROM repo ORDER BY repo_id DESC
            """)
        rs = pd.read_sql(repoUrlSQL, self.db, params={}).to_records()
        pop_off = 275
        i = 0
        while i < pop_off:
            rs = rs[1:]
            i += 1
        for row in rs:
            self._queue.put({'repo_id': row['repo_id'], 'repo_git': row['repo_git']})
        self.run()
        # self.discover_insights({'repo_id': 21420, 'repo_git': 'https://github.com/rails/ruby-coffee-script.git'})

    def update_config(self, config):
        """ Method to update config and set a default
{'fraction_non_alphanumeric': 0.11328422656845313, 'fraction_numerical': 0.004318008636017272, 'mean_word_length': 2.3769296740994856, 'pattern_counts': {'":': 12, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1084758', 'n_tokens_mistral': 2329, 'n_tokens_neox': 2226, 'n_words': 628}