<commit_before>import socket
import threading
from datetime import datetime

import pymongo
from upol_crawler.settings import *
from upol_crawler.tools import logger
from upol_crawler.utils import urls

log = logger.universal_logger('limiter')


def insert_limits_for_ip(db, domain, ip, last, max_frequency):
    """Insert limits for specific IP"""
    result = db['Limiter'].insert_one({'ip': ip,
                                       'domain': domain,
                                       'last': str(last),
                                       'max_frequency': max_frequency})

    return result is not None


def get_limits_for_ip(db, ip):
    """Return limits informations for specific IP"""
    result = db['Limiter'].find_one({'ip': ip})

    return result


def set_last_for_ip_limit(db, ip, last):
    """Set the last property for specific IP"""
    result = db['Limiter'].update({'ip': ip},
                                  {'$set':
                                   {'last': str(last)}})

    return result is not None


def _time_difference_from_now(time):
    """Return time difference from time to now"""
    delta = datetime.now() - time
    return delta.total_seconds()


def get_ip(url):
    """Return IP of website from URL"""
    domain = urls.domain(url)
    return socket.gethostbyname(domain)


def is_crawl_allowed(url):
    """Check if crawler is allowed to crawl given URL"""
    client = pymongo.MongoClient(
      CONFIG.get('Database', 'db_server'),
      int(CONFIG.get('Database', 'db_port')),
      maxPoolSize=None)
    database = client[DATABASE_NAME]

    ip = get_ip(url)

    result = True
    record = get_limits_for_ip(database, ip)

    if record is not None:
        last = datetime.strptime(record['last'], '%Y-%m-%d %H:%M:%S:%f')
        delta = _time_difference_from_now(last)

        if delta < float(record['max_frequency']):
            result = False
        else:
            set_last_for_ip_limit(database, ip, datetime.now())

    else:
        insert_limits_for_ip(database,
                             urls.domain(url),
                             ip,
                             datetime.now(),
                             float(CONFIG.get('Settings', 'crawl_frequency_per_server')))<commit_msg>In case of multiple time formats<commit_after>import socket
import threading
from datetime import datetime

import pymongo
from upol_crawler.settings import *
from upol_crawler.tools import logger
from upol_crawler.utils import urls

log = logger.universal_logger('limiter')


def insert_limits_for_ip(db, domain, ip, last, max_frequency):
    """Insert limits for specific IP"""
    result = db['Limiter'].insert_one({'ip': ip,
                                       'domain': domain,
                                       'last': str(last),
                                       'max_frequency': max_frequency})

    return result is not None


def get_limits_for_ip(db, ip):
    """Return limits informations for specific IP"""
    result = db['Limiter'].find_one({'ip': ip})

    return result


def set_last_for_ip_limit(db, ip, last):
    """Set the last property for specific IP"""
    result = db['Limiter'].update({'ip': ip},
                                  {'$set':
                                   {'last': str(last)}})

    return result is not None


def _time_difference_from_now(time):
    """Return time difference from time to now"""
    delta = datetime.now() - time
    return delta.total_seconds()


def get_ip(url):
    """Return IP of website from URL"""
    domain = urls.domain(url)
    return socket.gethostbyname(domain)


def is_crawl_allowed(url):
    """Check if crawler is allowed to crawl given URL"""
    client = pymongo.MongoClient(
      CONFIG.get('Database', 'db_server'),
      int(CONFIG.get('Database', 'db_port')),
      maxPoolSize=None)
    database = client[DATABASE_NAME]

    ip = get_ip(url)

    result = True
    record = get_limits_for_ip(database, ip)
    
    if record is not None:
        try:
            last = datetime.strptime(record['last'], '%Y-%m-%d %H:%M:%S.%f')
        except ValueError as e:
            last = datetime.strptime(record['last'], '%Y-%m-%d %H:%M:%S')

        delta = _time_difference_from_now(last)

        if delta < float(record['max_frequency']):
            result = False
        else:
            set_last_for_ip_limit(database, ip, datetime.now())

    else:
        insert_limits_for_ip(database,
                             urls.domain(url),
                             ip,
                             datetime.now(),
                             float(CONFIG.get('Settings', 'crawl_frequency_per_server')))
{'fraction_non_alphanumeric': 0.11592029140775659, 'fraction_numerical': 0.0, 'mean_word_length': 2.203843514070007, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4124634', 'n_tokens_mistral': 1377, 'n_tokens_neox': 1332, 'n_words': 341}