<commit_before>import mysql.connector as sqlconn
import logging
import settings
import sys
import socket

logger = logging.getLogger(__name__)


def results(url, status, exception, size, content_type, encoding, body, new_urls):
    ret = dict()
    ret['url'] = url
    ret['status'] = status
    ret['exception'] = exception
    ret['size'] = size
    ret['content_type'] = content_type
    ret['encoding'] = encoding
    ret['body'] = body
    ret['new_urls'] = new_urls
    return ret
    
FetchStatistic = results
                             
                             
def connect():
    try:
        db_conn = sqlconn.connect(host = settings.db_host,
                                  port = settings.db_port,
                                  user = settings.db_user,
                                  password = <PASSWORD>,
                                  database = settings.db_name)
    except:
        logger.error(sys.exc_info())
        quit()
    return db_conn
        
        
def get_tasks():
    tasks = processSQL('select id, url from {} where is_new=%s'.format(settings.task_table), (True,))
    return tasks
        
def get_crawlers():
    crawlers = processSQL('select id, addr_ip, addr_port from {} \
                            where running=%s order by id, count'.format(settings.crawler_table), (True,))
    return crawlers
 
def process_sql(sql, arg=None):
    db_conn = connect()
    cursor = db_conn.cursor()<commit_msg>nothing<commit_after>import mysql.connector as sqlconn
import logging
import settings
import sys
import socket

logger = logging.getLogger(__name__)

                             
                             
def connect():
    try:
        db_conn = sqlconn.connect(host = settings.db_host,
                                  port = settings.db_port,
                                  user = settings.db_user,
                                  password = <PASSWORD>,
                                  database = settings.db_name)
    except:
        logger.error(sys.exc_info())
        quit()
    return db_conn
        
        
def get_tasks():
    tasks = processSQL('select id, url from {} where is_new=%s'.format(settings.task_table), (True,))
    return tasks
        
def get_crawlers():
    crawlers = processSQL('select id, addr_ip, addr_port from {} \
                            where running=%s order by id, count'.format(settings.crawler_table), (True,))
    return crawlers
 
def process_sql(sql, arg=None):
    db_conn = connect()
    cursor = db_conn.cursor()
{'fraction_non_alphanumeric': 0.0978689818468824, 'fraction_numerical': 0.0, 'mean_word_length': 1.7796052631578947, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6901278', 'n_tokens_mistral': 687, 'n_tokens_neox': 678, 'n_words': 168}