<commit_before>"""
Warns about large files
"""
from __future__ import unicode_literals, print_function, division
import requests


def handle_url(bot, user, channel, url, msg):
    """inform about large files (over 5MB)"""

    # TODO: Use reference from bot / config
    browser = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.95 Safari/537.11"

    s = requests.session()
    s.stream = True  # Don't fetch content unless asked
    s.headers.update({'User-Agent': browser})

    # Be quiet on errors as this is not that important
    try:
        r = s.get(url)
    except:
        return None

    size = int(r.headers.get('Content-Length', 0))
    content_type = r.headers.get('content-type', None)
    if not content_type:
        content_type = "Unknown"
    size = size / 1024 / 1024

    # report files over 5 MB
    if size > 5:
        return bot.say(channel, "File size: %s MB - Content-Type: %s" % (int(size), content_type))<commit_msg>Removed useless TODO item and cleanup.<commit_after>"""
Warns about large files
"""
from __future__ import unicode_literals, print_function, division
import requests


def handle_url(bot, user, channel, url, msg):
    """inform about large files (over 5MB)"""
    browser = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.95 Safari/537.11"
    s = requests.session()
    s.stream = True  # Don't fetch content unless asked
    s.headers.update({'User-Agent': browser})
    # Be quiet on errors as this is not that important
    try:
        r = s.get(url)
    except:
        return None
    size = int(r.headers.get('Content-Length', 0))
    content_type = r.headers.get('content-type', None)
    if not content_type:
        content_type = "Unknown"
    size = size / 1024 / 1024
    # report files over 5 MB
    if size > 5:
        return bot.say(channel, "File size: %s MB - Content-Type: %s" % (int(size), content_type))
{'fraction_non_alphanumeric': 0.12309257375381485, 'fraction_numerical': 0.03763987792472025, 'mean_word_length': 3.0224948875255624, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6324212', 'n_tokens_mistral': 693, 'n_tokens_neox': 627, 'n_words': 204}