<commit_before>import urllib2
import urllib
import shutil
import logging
import hashlib
import socket
import binascii
import StringIO

def fetch_url(url, timeout=60.0, chunk_size=16384):
    req = urllib2.urlopen(url, timeout = timeout)
    strio = StringIO.StringIO()
    shutil.copyfileobj(req, strio, chunk_size)
    value = strio.getvalue()
    strio.close()
    return value


def fetch_imap():
    '''__placeholder'''


def fetch_pgp():
    '''__placeholder'''


def hashgen(data, func=hashlib.sha1):
    result = func()
    result.update(data)
    return result.hexdigest()


def log(logs_path, name, loglevel="DEBUG"):
    logger = logging.getLogger(name)
    logger.setLevel(loglevel)
    
    handler = logging.FileHandler("%s/%s.log" % (logs_path, name))
    handler.setLevel(loglevel)
    
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    handler.setFormatter(formatter)
    
    logger.addHandler(handler)    
    return logger  


def post_url(url, data, timeout=60.0, chunk_size=16384):
    encoded_data = urllib.urlencode(data)
    req = urllib2.urlopen(url, encoded_data, timeout)
    return req.read()

def is_ip(ip):
    try:
        ip_integer = socket.inet_pton(socket.AF_INET, ip)
        ip_version = 4
    except socket.error:
        try:
            ip_integer = socket.inet_pton(socket.AF_INET6, ip)
            ip_version = 6
        except socket.error:
            return None
        
    ip_integer = int(binascii.hexlify(ip_integer), 16)
    return [ ip , ip_version, ip_integer ]

def reverse_ip(ip):
    import dns.reversename
    result = str(dns.reversename.from_address(ip))
    reverse = result.split('.in-addr.arpa.')
    if not reverse:
        reverse = result.split('.ip6.arpa.')
    return reverse[0]


def decode(text, encodings=["ascii", "utf-8"]):
    if isinstance(text, unicode):
        return unicode(text)

    for encoding in encodings:
        try:
            return unicode(text, encoding)
        except ValueError as e:
            pass

    return None 
<commit_msg>Update utils.py<commit_after>import urllib2
import urllib
import shutil
import logging
import hashlib
import socket
import binascii
import StringIO

def fetch_url(url, timeout=60.0, chunk_size=16384):
    req = urllib2.urlopen(url, timeout = timeout)
    strio = StringIO.StringIO()
    shutil.copyfileobj(req, strio, chunk_size)
    value = strio.getvalue()
    strio.close()
    return value


def fetch_imap():
    '''__placeholder'''


def fetch_pgp():
    '''__placeholder'''


def hashgen(data, func=hashlib.sha1):
    result = func()
    result.update(data)
    return result.hexdigest()


def log(logs_path, name, loglevel="DEBUG"):
    logger = logging.getLogger(name)
    logger.setLevel(loglevel)
    
    handler = logging.FileHandler("%s/%s.log" % (logs_path, name))
    handler.setLevel(loglevel)
    
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    handler.setFormatter(formatter)
    
    logger.addHandler(handler)    
    return logger  


def post_url(url, data, timeout=60.0, chunk_size=16384):
    encoded_data = urllib.urlencode(data)
    req = urllib2.urlopen(url, encoded_data, timeout)
    return req.read()

def is_ip(ip):
    try:
        ip_integer = socket.inet_pton(socket.AF_INET, ip)
        ip_version = 4
    except socket.error:
        try:
            ip_integer = socket.inet_pton(socket.AF_INET6, ip)
            ip_version = 6
        except socket.error:
            return None
        
    ip_integer = int(binascii.hexlify(ip_integer), 16)
    return [ ip , ip_version, ip_integer ]

def reverse_ip(ip):
    import dns.reversename
    result = str(dns.reversename.from_address(ip))
    reverse = result.split('.in-addr.arpa.')
    if not reverse:
        reverse = result.split('.ip6.arpa.')
    return reverse[0]


def decode(text, encodings=["ascii", "utf-8"]):
    try:
        return unicode(text)
    except ValueError as e:
        pass

    for encoding in encodings:
        try:
            return unicode(text, encoding)
        except ValueError as e:
            pass

    return None 

{'fraction_non_alphanumeric': 0.11016166281755196, 'fraction_numerical': 0.012933025404157044, 'mean_word_length': 2.4871175523349436, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '617478', 'n_tokens_mistral': 1515, 'n_tokens_neox': 1388, 'n_words': 300}