<commit_before>import pymongo
from crawler.urls import url_tools
from bson import json_util
import json

def get_log_format(response):
    log = {}

    formated_headers = {}

    for key, value in response.headers.items():
        formated_headers[str(key)] = str(value)

    log['encoding'] = response.encoding
    log['elapsed'] = str(response.elapsed)
    log['headers'] = formated_headers
    log['is_permanent_redirect'] = response.is_permanent_redirect
    log['is_redirect'] = response.is_redirect
    log['status_code'] = response.status_code
    log['reason'] = response.reason

    return log


def log_url(url, response):
    client = pymongo.MongoClient('localhost', 27017)
    database = client.upol_crawler

    log_object = {"_id": url_tools.hash(url),
                  "url": url,
                  "encoding": response.get('encoding'),
                  "elapsed": response.get('elapsed'),
                  "headers": response.get('headers'),
                  "is_permanent_redirect": response.get('is_permanent_redirect'),
                  "is_redirect": response.get('is_redirect'),
                  "status_code": response.get('status_code'),
                  "reason": response.get('reason')}

    try:
        database.urls_logs.insert_one(log_object).inserted_id
    except pymongo.errors.DuplicateKeyError as e:
        return False


    client.close()

def log_url_reason(url, reason, arg={}):
    client = pymongo.MongoClient('localhost', 27017)
    database = client[reason]

    log_object = {"_id": url_tools.hash(url),
                  "url": url}

    for key, value in arg.items():
        log_object[key] = str(value)

    try:
        database.urls_logs_not_valid.insert_one(log_object).inserted_id
    except pymongo.errors.DuplicateKeyError as e:
        return False

    client.close()


# Deprecated
def log_url_validator(url, validator, arg=None):
    client = pymongo.MongoClient('localhost', 27017)
    database = client.upol_crawler

    if validator == "blacklist":
        log_object = {
                      "url": url,
                      "blacklisted": True}

    elif validator == "robots_block":
        log_object = {
                      "url": url,
                      "robots_block": True}

    elif validator == "exception":
        log_object = {<commit_msg>Insert into right collection<commit_after>import pymongo
from crawler.urls import url_tools
from bson import json_util
import json

def get_log_format(response):
    log = {}

    formated_headers = {}

    for key, value in response.headers.items():
        formated_headers[str(key)] = str(value)

    log['encoding'] = response.encoding
    log['elapsed'] = str(response.elapsed)
    log['headers'] = formated_headers
    log['is_permanent_redirect'] = response.is_permanent_redirect
    log['is_redirect'] = response.is_redirect
    log['status_code'] = response.status_code
    log['reason'] = response.reason

    return log


def log_url(url, response):
    client = pymongo.MongoClient('localhost', 27017)
    database = client.upol_crawler

    log_object = {"_id": url_tools.hash(url),
                  "url": url,
                  "encoding": response.get('encoding'),
                  "elapsed": response.get('elapsed'),
                  "headers": response.get('headers'),
                  "is_permanent_redirect": response.get('is_permanent_redirect'),
                  "is_redirect": response.get('is_redirect'),
                  "status_code": response.get('status_code'),
                  "reason": response.get('reason')}

    try:
        database.urls_logs.insert_one(log_object).inserted_id
    except pymongo.errors.DuplicateKeyError as e:
        return False


    client.close()

def log_url_reason(url, reason, arg={}):
    client = pymongo.MongoClient('localhost', 27017)
    database = client.upol_crawler
    collection = database[reason]

    log_object = {"_id": url_tools.hash(url),
                  "url": url}

    for key, value in arg.items():
        log_object[key] = str(value)

    try:
        collection.insert_one(log_object).inserted_id
    except pymongo.errors.DuplicateKeyError as e:
        return False

    client.close()


# Deprecated
def log_url_validator(url, validator, arg=None):
    client = pymongo.MongoClient('localhost', 27017)
    database = client.upol_crawler

    if validator == "blacklist":
        log_object = {
                      "url": url,
                      "blacklisted": True}

    elif validator == "robots_block":
        log_object = {
                      "url": url,
                      "robots_block": True}

    elif validator == "exception":
        log_object = {
{'fraction_non_alphanumeric': 0.1178099701746911, 'fraction_numerical': 0.006391137622496805, 'mean_word_length': 2.5274229902329077, 'pattern_counts': {'":': 36, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3990001', 'n_tokens_mistral': 1454, 'n_tokens_neox': 1376, 'n_words': 275}