<commit_before># -*- coding: utf-8 -*-
"""Show info about posted URLs

"""

import re
import pickle
import socket
from urllib.parse import urlparse

from bs4 import BeautifulSoup
import requests
import requests.exceptions
from irc3 import plugin, event


def sizeof_fmt(num, suffix='B'):
    for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']:
        if abs(num) < 1024.0:
            return "%3.1f%s%s" % (num, unit, suffix)
        num /= 1024.0
    return "%.1f%s%s" % (num, 'Yi', suffix)


@plugin
class UrlInfo(object):
    """Bot User Interface plugin"""

    def __init__(self, bot):
        """Init"""
        self.bot = bot
        self.config = bot.config.get(__name__, {})
        self.log = self.bot.log.getChild(__name__)
        cookiejar_file = self.config.get('cookiejar')
        if cookiejar_file:
            with open(cookiejar_file, 'rb') as f:
                self.cookiejar = pickle.load(f)

        self.urlmap = self.bot.config.get(__name__ + '.urlmap', {})

    @event('^:(?P<mask>\S+!\S+@\S+) (?P<event>(PRIVMSG|NOTICE)) '
           '(?P<target>\S+) :\s*(?P<data>(.*(https?://)).*)$')
    def on_message(self, mask, event, target, data):
        if mask.nick == self.bot.nick or event == 'NOTICE':
            return
        index = 1
        urls = re.findall(r'https?://\S+', data)
        for url in urls:
            message = ["URL:"]
            o = urlparse(url)
            if o.hostname in ('localhost', '127.0.0.1', socket.gethostname()):
                continue
            if o.hostname in self.urlmap:
                url = url.replace(o.hostname, self.urlmap[o.hostname], 1)
                message.append(url)
            with requests.Session() as session:<commit_msg>add some more empty hosts<commit_after># -*- coding: utf-8 -*-
"""Show info about posted URLs

"""

import re
import pickle
import socket
from urllib.parse import urlparse

from bs4 import BeautifulSoup
import requests
import requests.exceptions
from irc3 import plugin, event


def sizeof_fmt(num, suffix='B'):
    for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']:
        if abs(num) < 1024.0:
            return "%3.1f%s%s" % (num, unit, suffix)
        num /= 1024.0
    return "%.1f%s%s" % (num, 'Yi', suffix)


@plugin
class UrlInfo(object):
    """Bot User Interface plugin"""

    def __init__(self, bot):
        """Init"""
        self.bot = bot
        self.config = bot.config.get(__name__, {})
        self.log = self.bot.log.getChild(__name__)
        cookiejar_file = self.config.get('cookiejar')
        if cookiejar_file:
            with open(cookiejar_file, 'rb') as f:
                self.cookiejar = pickle.load(f)

        self.urlmap = self.bot.config.get(__name__ + '.urlmap', {})

    @event('^:(?P<mask>\S+!\S+@\S+) (?P<event>(PRIVMSG|NOTICE)) '
           '(?P<target>\S+) :\s*(?P<data>(.*(https?://)).*)$')
    def on_message(self, mask, event, target, data):
        if mask.nick == self.bot.nick or event == 'NOTICE':
            return
        index = 1
        urls = re.findall(r'https?://\S+', data)
        for url in urls:
            message = ["URL:"]
            o = urlparse(url)
            if o.hostname in ('127.0.0.1', '[::1]',
                              'localhost',
                              'localhost.localdomain',
                              socket.gethostname()):
                continue
            if o.hostname in self.urlmap:
                url = url.replace(o.hostname, self.urlmap[o.hostname], 1)
                message.append(url)
            with requests.Session() as session:
{'fraction_non_alphanumeric': 0.16675977653631285, 'fraction_numerical': 0.013687150837988826, 'mean_word_length': 2.087068965517241, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '484422', 'n_tokens_mistral': 1229, 'n_tokens_neox': 1172, 'n_words': 285}