<commit_before>import apscheduler.schedulers.background
import datetime
import distutils.spawn
import importlib
import json
import mysql.connector
import queue
import time
import sys
import pprint
from utilities import url_to_json

tokens = []


class Tokenizer():
    def __init__(self):
        self.have_tokens = bool(tokens)
        self.available_tokens = queue.Queue()
        self.scheduler = apscheduler.schedulers.background.BackgroundScheduler(
            )
        self.scheduler.start()

        if not self.have_tokens:
            self.print_warning(
                'No GitHub OAuth tokens provided. Proceeding without '
                'authentication.'
            )

        for token in tokens:
            self.available_tokens.put(token)

    def tokenize(self, url):
        if url.startswith('https://api.github.com'):
            if self.have_tokens:
                token = self.get_token()
                if token is not None:
                    return '{0}?access_token={1}'.format(url, token)
                else:
                    return url
            else:
                return url
        else:
            raise ValueError('url must be for the GitHub API')

    def get_token(self):
        while True:
            if not self.scheduler.get_jobs() and self.available_tokens.empty():
                self.print_warning('No more valid OAuth tokens available.')
                return None

            token = self.available_tokens.get(block=True)

            rate_limit_url = (
                'https://api.github.com/rate_limit?access_token={0}'
            ).format(token)
            status = url_to_json(rate_limit_url)

            # Throw away bad OAuth keys.
            if 'resources' not in status:
                self.print_warning(
                    'Invalid OAuth token supplied. Trying again...'
                )
                continue

            if status['resources']['core']['remaining'] > 0:
                self.available_tokens.put_nowait(token)
                return token
            else:
                self.scheduler.add_job(
                    self.available_tokens.put_nowait,
                    'date',
                    args=[token],
                    run_date=datetime.datetime.fromtimestamp(
                        status['resources']['core']['reset']
                    )
                )

    def print_warning(self, message):
        formatted_message = '\033[91mWARNING\033[0m: {0}'.format(message)
        print(formatted_message)


def get_run_id(cursor):
    '''
    Generate an id from the datasource that uniquely identifies the run.

    Args:
        cursor: mysql.cursor.MySQLCursor
            Cursor object used to insert data.

    Return:
        Identifier of the run to persist.
    '''
    cursor.execute('INSERT INTO reaper_runs VALUES()')

    return cursor.lastrowid


def save_result(run_id, repo_id, results, cursor):
    """
    Save the results to the specified data source, creating the results table
    as necessary.

    Args:
        run_id: int
            Identifier of the run to persist.
        repo_id: int
            Identifier of the repository to save.
        results: dict
            Key value pair of results to be saved.
        cursor: mysql.cursor.MySQLCursor
            Cursor object used to insert data.

    Return:
        True if successful, False otherwise.
    """
    query = '''
        INSERT INTO reaper_results
            (project_id, results, run_id)
        VALUES
            ({0}, '{1}', {2})
        '''.format(repo_id, json.dumps(results), run_id)
    
    cursor.execute(query)


def init_attribute_plugins(attributes, connection):
    for attribute in attributes:
        if 'implementation' in attribute:
            try:
                cursor = connection.cursor()
                attribute['implementation'].init(cursor)
            except:
                pass
            finally:
                cursor.close()


def global_init_attribute_plugins(attributes, connection, sample):
    for attribute in attributes:
        if 'implementation' in attribute:
            try:
                cursor = connection.cursor()
                attribute['implementation'].global_init(cursor, sample)
            except:
                pass
            finally:
                cursor.close()


def process_repository(project_id, repo_path, attributes, connection):
    """
    Calculate a score for the given repository.

    Args:
        project_id: int
            GHTorrent dataset identifier for the repository.
        repo_path: string
            Path to the repository contents.
        attributes: list
            List of attributes to be executed against the repository.
        connection: mysql.connector.MySQLConnection
            Database connection used for querying the GHTorrent dataset.
    """
    score = 0
    results = {}
    for attribute in attributes:
        if 'implementation' in attribute:
            cursor = connection.cursor()

            binary_result, raw_result = attribute['implementation'].run(
                project_id,
                repo_path,
                cursor,
                **attribute.get('options', {})
            )

            cursor.close()

            score += binary_result * attribute['weight']
            results[attribute['name']] = raw_result

            if ('essential' in attribute and<commit_msg>Fix issue related to long running processes<commit_after>import apscheduler.schedulers.background
import datetime
import distutils.spawn
import importlib
import json
import mysql.connector
import queue
import time
import sys
import pprint
from utilities import url_to_json

tokens = []


class Tokenizer():
    def __init__(self):
        self.have_tokens = bool(tokens)
        self.available_tokens = queue.Queue()
        self.scheduler = apscheduler.schedulers.background.BackgroundScheduler(
            )
        self.scheduler.start()

        if not self.have_tokens:
            self.print_warning(
                'No GitHub OAuth tokens provided. Proceeding without '
                'authentication.'
            )

        for token in tokens:
            self.available_tokens.put(token)

    def tokenize(self, url):
        if url.startswith('https://api.github.com'):
            if self.have_tokens:
                token = self.get_token()
                if token is not None:
                    return '{0}?access_token={1}'.format(url, token)
                else:
                    return url
            else:
                return url
        else:
            raise ValueError('url must be for the GitHub API')

    def get_token(self):
        while True:
            if not self.scheduler.get_jobs() and self.available_tokens.empty():
                self.print_warning('No more valid OAuth tokens available.')
                return None

            token = self.available_tokens.get(block=True)

            rate_limit_url = (
                'https://api.github.com/rate_limit?access_token={0}'
            ).format(token)
            status = url_to_json(rate_limit_url)

            # Throw away bad OAuth keys.
            if 'resources' not in status:
                self.print_warning(
                    'Invalid OAuth token supplied. Trying again...'
                )
                continue

            if status['resources']['core']['remaining'] > 0:
                self.available_tokens.put_nowait(token)
                return token
            else:
                self.scheduler.add_job(
                    self.available_tokens.put_nowait,
                    'date',
                    args=[token],
                    run_date=datetime.datetime.fromtimestamp(
                        status['resources']['core']['reset']
                    )
                )

    def print_warning(self, message):
        formatted_message = '\033[91mWARNING\033[0m: {0}'.format(message)
        print(formatted_message)


def get_run_id(cursor):
    '''
    Generate an id from the datasource that uniquely identifies the run.

    Args:
        cursor: mysql.cursor.MySQLCursor
            Cursor object used to insert data.

    Return:
        Identifier of the run to persist.
    '''
    cursor.execute('INSERT INTO reaper_runs VALUES()')

    return cursor.lastrowid


def save_result(run_id, repo_id, results, cursor):
    """
    Save the results to the specified data source, creating the results table
    as necessary.

    Args:
        run_id: int
            Identifier of the run to persist.
        repo_id: int
            Identifier of the repository to save.
        results: dict
            Key value pair of results to be saved.
        cursor: mysql.cursor.MySQLCursor
            Cursor object used to insert data.

    Return:
        True if successful, False otherwise.
    """
    query = '''
        INSERT INTO reaper_results
            (project_id, results, run_id)
        VALUES
            ({0}, '{1}', {2})
        '''.format(repo_id, json.dumps(results), run_id)
    
    cursor.execute(query)


def init_attribute_plugins(attributes, connection):
    for attribute in attributes:
        if 'implementation' in attribute:
            try:
                cursor = connection.cursor()
                attribute['implementation'].init(cursor)
            except:
                pass
            finally:
                cursor.close()


def global_init_attribute_plugins(attributes, connection, sample):
    for attribute in attributes:
        if 'implementation' in attribute:
            try:
                cursor = connection.cursor()
                attribute['implementation'].global_init(cursor, sample)
            except:
                pass
            finally:
                cursor.close()


def process_repository(project_id, repo_path, attributes, connection):
    """
    Calculate a score for the given repository.

    Args:
        project_id: int
            GHTorrent dataset identifier for the repository.
        repo_path: string
            Path to the repository contents.
        attributes: list
            List of attributes to be executed against the repository.
        connection: mysql.connector.MySQLConnection
            Database connection used for querying the GHTorrent dataset.
    """
    score = 0
    results = {}
    for attribute in attributes:
        if 'implementation' in attribute:
            try:
                cursor = connection.cursor()
            except mysql.connector.Error:
                sys.stderr.write(
                    'Error while creating cursor, attempting to reconnect.\n'
                )
                connection.reconnect()
            finally:
                cursor = connection.cursor()

            binary_result, raw_result = attribute['implementation'].run(
                project_id,
                repo_path,
                cursor,
                **attribute.get('options', {})
            )

            cursor.close()

            score += binary_result * attribute['weight']
            results[attribute['name']] = raw_result

            if ('essential' in attribute and
{'fraction_non_alphanumeric': 0.08035395066142295, 'fraction_numerical': 0.0032177332856632105, 'mean_word_length': 1.7013520038628682, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1471628', 'n_tokens_mistral': 2867, 'n_tokens_neox': 2677, 'n_words': 774}