<commit_before># -*- coding: utf-8 -*-
# vim: set ts=4 et

import cgi
import requests
from six.moves.html_parser import HTMLParser

from plugin import *


content_types = (
    'text/html',
    'text/xml',
    'application/xhtml+xml',
    'application/xml'
)

class TitleParser(HTMLParser):
    def __init__(self):
        HTMLParser.__init__(self)
        self.match = False
        self.title = ''
    
    def handle_starttag(self, tag, attrs):
        if tag == 'meta':
            og_title = False
            for attr in attrs:
                if attr == ('property', 'og:title'):
                    og_title = True
            if og_title:
                for attr in attrs:
                    if attr[0] == 'content':
                        self.title = attr[1]
        
        self.match = True if not self.title and tag == 'title' else False
        
    def handle_data(self, data):
        if self.match:
            self.title = data.strip()
            self.match = False


class Plugin(BasePlugin):
    default_priority = 1
    
    @hook
    def any_url(self, msg, domain, url):
        headers = {
            'User-Agent': 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Safari/537.36'
        }
        r = requests.get(url, stream=True, headers=headers)

        content_type, params = cgi.parse_header(r.headers['Content-Type'])
        if not content_type in content_types:
            return

        r.encoding = 'utf-8'
        if 'charset' in params:<commit_msg>Added time out to anyurl request.<commit_after># -*- coding: utf-8 -*-
# vim: set ts=4 et

import cgi
import requests
from six.moves.html_parser import HTMLParser

from plugin import *


content_types = (
    'text/html',
    'text/xml',
    'application/xhtml+xml',
    'application/xml'
)

class TitleParser(HTMLParser):
    def __init__(self):
        HTMLParser.__init__(self)
        self.match = False
        self.title = ''
    
    def handle_starttag(self, tag, attrs):
        if tag == 'meta':
            og_title = False
            for attr in attrs:
                if attr == ('property', 'og:title'):
                    og_title = True
            if og_title:
                for attr in attrs:
                    if attr[0] == 'content':
                        self.title = attr[1]
        
        self.match = True if not self.title and tag == 'title' else False
        
    def handle_data(self, data):
        if self.match:
            self.title = data.strip()
            self.match = False


class Plugin(BasePlugin):
    default_priority = 1
    
    @hook
    def any_url(self, msg, domain, url):
        headers = {
            'User-Agent': 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Safari/537.36'
        }

        try:
            r = requests.get(url, stream=True, headers=headers, timeout=10)
        except requests.exceptions.ReadTimeout as e:
            msg.reply('URL Timeout')
            return

        content_type, params = cgi.parse_header(r.headers['Content-Type'])
        if not content_type in content_types:
            return

        r.encoding = 'utf-8'
        if 'charset' in params:
{'fraction_non_alphanumeric': 0.11311525527361663, 'fraction_numerical': 0.012840110058086213, 'mean_word_length': 1.8158347676419966, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1272015', 'n_tokens_mistral': 1009, 'n_tokens_neox': 970, 'n_words': 253}