<commit_before>import re

from BeautifulSoup import BeautifulSoup # use BS to parse HTML (it's easy!)
import lxml.html # use lxml.html to parse HTML (it's fast!)
import StringIO

import oembed
from oembed.constants import OEMBED_BLOCK_ELEMENTS, URL_RE, STANDALONE_URL_RE
from oembed.exceptions import OEmbedException
from oembed.parsers.base import BaseParser
from oembed.parsers.text import TextParser, TextBlockParser


class HTMLParser(BaseParser):
    """
    Use BeautifulSoup for easy html processing.
    """
    def parse_data(self, text, maxwidth, maxheight, template_dir, context,
                   urlize_all_links):                
        block_parser = TextBlockParser()
        original_template_dir = template_dir
        
        soup = BeautifulSoup(text)
        
        for user_url in soup.findAll(text=re.compile(URL_RE)):
            if not self.inside_a(user_url):
                if self.is_standalone(user_url):
                    template_dir = original_template_dir
                else:
                    template_dir = 'inline'
                
                replacement = block_parser.parse(
                    str(user_url),
                    maxwidth,
                    maxheight,
                    template_dir,
                    context,
                    urlize_all_links
                )
                user_url.replaceWith(replacement)
        
        return unicode(soup)
    
    def is_standalone(self, soupie):
        if re.match(STANDALONE_URL_RE, soupie):
            if soupie.parent.name in OEMBED_BLOCK_ELEMENTS:
                return True
        return False
    
    def inside_a(self, soupie):
        parent = soupie.parent
        while parent is not None:
            if parent.name == 'a':
                return True
            parent = parent.parent
        return False
    
    def extract_urls(self, text):
        block_parser = TextBlockParser()
        soup = BeautifulSoup(text)
        urls = set()
        
        for user_url in soup.findAll(text=re.compile(URL_RE)):
            if not self.inside_a(user_url):
                urls |= block_parser.extract_urls(unicode(user_url))
        
        return urls


class LXMLParser(BaseParser):
    """
    Use lxml.html, and lxml.etree for fast html processing.  This feature
    is not fully implemented yet.
    """
    def parse_data(self, text, maxwidth, maxheight, template_dir, context,
                   urlize_all_links):
        block_parser = TextBlockParser()
        text_parser = TextParser()
        
        try:
            parse_tree = lxml.html.fragment_fromstring(text, create_parent='div')
        except lxml.etree.XMLSyntaxError:
            return text
        
        if not parse_tree.getchildren():
            elements = [parse_tree]
        else:
            elements = parse_tree.xpath('.//*[not(self::a) and contains(text(), "http://")]')
        
        for element in elements:
            replacement = block_parser.parse(
                element.text,
                maxwidth,
                maxheight,
                template_dir,
                context,
                urlize_all_links
            )
            if replacement != element.text:
                element.text = ''
                new_elements = lxml.html.fragments_fromstring(replacement)
                for (i, e) in enumerate(new_elements):
                    element.insert(i, e)
        
        return lxml.html.tostring(parse_tree)<commit_msg>Removing the lxml parser which was not implemented fully<commit_after>import re

from BeautifulSoup import BeautifulSoup # use BS to parse HTML (it's easy!)

import oembed
from oembed.constants import OEMBED_BLOCK_ELEMENTS, URL_RE, STANDALONE_URL_RE
from oembed.exceptions import OEmbedException
from oembed.parsers.base import BaseParser
from oembed.parsers.text import TextParser, TextBlockParser


class HTMLParser(BaseParser):
    """
    Use BeautifulSoup for easy html processing.
    """
    def parse_data(self, text, maxwidth, maxheight, template_dir, context,
                   urlize_all_links):                
        block_parser = TextBlockParser()
        original_template_dir = template_dir
        
        soup = BeautifulSoup(text)
        
        for user_url in soup.findAll(text=re.compile(URL_RE)):
            if not self.inside_a(user_url):
                if self.is_standalone(user_url):
                    template_dir = original_template_dir
                else:
                    template_dir = 'inline'
                
                replacement = block_parser.parse(
                    str(user_url),
                    maxwidth,
                    maxheight,
                    template_dir,
                    context,
                    urlize_all_links
                )
                user_url.replaceWith(replacement)
        
        return unicode(soup)
    
    def is_standalone(self, soupie):
        if re.match(STANDALONE_URL_RE, soupie):
            if soupie.parent.name in OEMBED_BLOCK_ELEMENTS:
                return True
        return False
    
    def inside_a(self, soupie):
        parent = soupie.parent
        while parent is not None:
            if parent.name == 'a':
                return True
            parent = parent.parent
        return False
    
    def extract_urls(self, text):
        block_parser = TextBlockParser()
        soup = BeautifulSoup(text)
        urls = set()
        
        for user_url in soup.findAll(text=re.compile(URL_RE)):
            if not self.inside_a(user_url):
                urls |= block_parser.extract_urls(unicode(user_url))
        
        return urls
{'fraction_non_alphanumeric': 0.07025884838880085, 'fraction_numerical': 0.0, 'mean_word_length': 1.7112171837708832, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 9}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5073123', 'n_tokens_mistral': 1568, 'n_tokens_neox': 1511, 'n_words': 361}