<commit_before>"""
Page parsing

Parsing of web pages for extraction task.
"""
from collections import defaultdict
from numpy import array

from scrapy.utils.py26 import json

from scrapy.contrib.ibl.htmlpage import HtmlTagType, HtmlTag, HtmlPage
from scrapy.contrib.ibl.extraction.pageobjects import (AnnotationTag, 
        TemplatePage, ExtractionPage, AnnotationText, TokenDict)

def parse_strings(template_html, extraction_html):
    """Create a template and extraction page from raw strings

    this is useful for testing purposes
    """
    t = TokenDict()
    template_page = HtmlPage(body=template_html)
    extraction_page = HtmlPage(body=extraction_html)
    return (parse_template(t, template_page), 
            parse_extraction_page(t, extraction_page))

def parse_template(token_dict, template_html):
    """Create an TemplatePage object by parsing the annotated html"""
    parser = TemplatePageParser(token_dict)
    parser.feed(template_html)
    return parser.to_template()

def parse_extraction_page(token_dict, page_html):
    """Create an ExtractionPage object by parsing the html"""
    parser = ExtractionPageParser(token_dict)
    parser.feed(page_html)
    return parser.to_extraction_page()

class InstanceLearningParser(object):
    """Base parser for instance based learning algorithm
    
    This does not require correct HTML and the parsing method should not alter
    the original tag order. It is important that parsing results do not vary.
    """
    def __init__(self, token_dict):
        self.token_dict = token_dict
        self.token_list = []
    
    def _add_token(self, token, token_type, start, end):
        tid = self.token_dict.tokenid(token, token_type)
        self.token_list.append(tid)

    def feed(self, html_page):
        self.html_page = html_page
        self.previous_element_class = None
        for data in html_page.parsed_body:
            if isinstance(data, HtmlTag):
                self._add_token(data.tag, data.tag_type, data.start, data.end)
                self.handle_tag(data)
            else:
                self.handle_data(data)
            self.previous_element_class = data.__class__

    def handle_data(self, html_data_fragment):
        pass

    def handle_tag(self, html_tag):
        pass

_END_UNPAIREDTAG_TAGS = ["form", "div", "p", "table", "tr", "td"]

class TemplatePageParser(InstanceLearningParser):
    """Template parsing for instance based learning algorithm"""

    def __init__(self, token_dict):
        InstanceLearningParser.__init__(self, token_dict)
        self.annotations = []
        self.ignored_regions = []
        self.extra_required_attrs = []
        self.ignored_tag_stacks = defaultdict(list)
        # tag names that have not been completed
        self.labelled_tag_stacks = defaultdict(list)
        self.replacement_stacks = defaultdict(list)
        self.unpairedtag_stack = []
        self.variant_stack = []
        self.prev_data = None
        self.last_text_region = None
        self.next_tag_index = 0

    def handle_tag(self, html_tag):
        if self.last_text_region:
            self._process_text('')
        
        if html_tag.tag_type == HtmlTagType.OPEN_TAG:
            self._handle_open_tag(html_tag)
        elif html_tag.tag_type == HtmlTagType.CLOSE_TAG:
            self._handle_close_tag(html_tag)
        else:
            # the tag is not paired, it can contain only attribute annotations
            self._handle_unpaired_tag(html_tag)
    
    @staticmethod
    def _read_template_annotation(html_tag):
        template_attr = html_tag.attributes.get('data-scrapy-annotate')
        if template_attr is None:
            return None
        unescaped = template_attr.replace('&quot;', '"')
        return json.loads(unescaped)
    
    @staticmethod
    def _read_bool_template_attribute(html_tag, attribute):
        return html_tag.attributes.get("data-scrapy-" + attribute) == "true"
    
    def _close_unpaired_tag(self):
        self.unpairedtag_stack[0].end_index = self.next_tag_index
        self.unpairedtag_stack = []

    def _handle_unpaired_tag(self, html_tag):
        if self._read_bool_template_attribute(html_tag, "ignore") and html_tag.tag == "img":
            self.ignored_regions.append((self.next_tag_index, self.next_tag_index + 1))
        elif self._read_bool_template_attribute(html_tag, "ignore-beneath") and html_tag.tag == "img":
            self.ignored_regions.append((self.next_tag_index, None))
        jannotation = self._read_template_annotation(html_tag)
        if jannotation:
            if self.unpairedtag_stack:
                self._close_unpaired_tag()
                
            annotation = AnnotationTag(self.next_tag_index, self.next_tag_index + 1)
            attribute_annotations = jannotation.pop('annotations', {}).items()
            for extract_attribute, tag_value in attribute_annotations:
                if extract_attribute == 'content':
                    annotation.surrounds_attribute = tag_value
                    self.unpairedtag_stack.append(annotation)<commit_msg>Remove restriction of marking ignore-beneath only for img unpaired tags<commit_after>"""
Page parsing

Parsing of web pages for extraction task.
"""
from collections import defaultdict
from numpy import array

from scrapy.utils.py26 import json

from scrapy.contrib.ibl.htmlpage import HtmlTagType, HtmlTag, HtmlPage
from scrapy.contrib.ibl.extraction.pageobjects import (AnnotationTag, 
        TemplatePage, ExtractionPage, AnnotationText, TokenDict)

def parse_strings(template_html, extraction_html):
    """Create a template and extraction page from raw strings

    this is useful for testing purposes
    """
    t = TokenDict()
    template_page = HtmlPage(body=template_html)
    extraction_page = HtmlPage(body=extraction_html)
    return (parse_template(t, template_page), 
            parse_extraction_page(t, extraction_page))

def parse_template(token_dict, template_html):
    """Create an TemplatePage object by parsing the annotated html"""
    parser = TemplatePageParser(token_dict)
    parser.feed(template_html)
    return parser.to_template()

def parse_extraction_page(token_dict, page_html):
    """Create an ExtractionPage object by parsing the html"""
    parser = ExtractionPageParser(token_dict)
    parser.feed(page_html)
    return parser.to_extraction_page()

class InstanceLearningParser(object):
    """Base parser for instance based learning algorithm
    
    This does not require correct HTML and the parsing method should not alter
    the original tag order. It is important that parsing results do not vary.
    """
    def __init__(self, token_dict):
        self.token_dict = token_dict
        self.token_list = []
    
    def _add_token(self, token, token_type, start, end):
        tid = self.token_dict.tokenid(token, token_type)
        self.token_list.append(tid)

    def feed(self, html_page):
        self.html_page = html_page
        self.previous_element_class = None
        for data in html_page.parsed_body:
            if isinstance(data, HtmlTag):
                self._add_token(data.tag, data.tag_type, data.start, data.end)
                self.handle_tag(data)
            else:
                self.handle_data(data)
            self.previous_element_class = data.__class__

    def handle_data(self, html_data_fragment):
        pass

    def handle_tag(self, html_tag):
        pass

_END_UNPAIREDTAG_TAGS = ["form", "div", "p", "table", "tr", "td"]

class TemplatePageParser(InstanceLearningParser):
    """Template parsing for instance based learning algorithm"""

    def __init__(self, token_dict):
        InstanceLearningParser.__init__(self, token_dict)
        self.annotations = []
        self.ignored_regions = []
        self.extra_required_attrs = []
        self.ignored_tag_stacks = defaultdict(list)
        # tag names that have not been completed
        self.labelled_tag_stacks = defaultdict(list)
        self.replacement_stacks = defaultdict(list)
        self.unpairedtag_stack = []
        self.variant_stack = []
        self.prev_data = None
        self.last_text_region = None
        self.next_tag_index = 0

    def handle_tag(self, html_tag):
        if self.last_text_region:
            self._process_text('')
        
        if html_tag.tag_type == HtmlTagType.OPEN_TAG:
            self._handle_open_tag(html_tag)
        elif html_tag.tag_type == HtmlTagType.CLOSE_TAG:
            self._handle_close_tag(html_tag)
        else:
            # the tag is not paired, it can contain only attribute annotations
            self._handle_unpaired_tag(html_tag)
    
    @staticmethod
    def _read_template_annotation(html_tag):
        template_attr = html_tag.attributes.get('data-scrapy-annotate')
        if template_attr is None:
            return None
        unescaped = template_attr.replace('&quot;', '"')
        return json.loads(unescaped)
    
    @staticmethod
    def _read_bool_template_attribute(html_tag, attribute):
        return html_tag.attributes.get("data-scrapy-" + attribute) == "true"
    
    def _close_unpaired_tag(self):
        self.unpairedtag_stack[0].end_index = self.next_tag_index
        self.unpairedtag_stack = []

    def _handle_unpaired_tag(self, html_tag):
        if self._read_bool_template_attribute(html_tag, "ignore") and html_tag.tag == "img":
            self.ignored_regions.append((self.next_tag_index, self.next_tag_index + 1))
        elif self._read_bool_template_attribute(html_tag, "ignore-beneath"):
            self.ignored_regions.append((self.next_tag_index, None))
        jannotation = self._read_template_annotation(html_tag)
        if jannotation:
            if self.unpairedtag_stack:
                self._close_unpaired_tag()
                
            annotation = AnnotationTag(self.next_tag_index, self.next_tag_index + 1)
            attribute_annotations = jannotation.pop('annotations', {}).items()
            for extract_attribute, tag_value in attribute_annotations:
                if extract_attribute == 'content':
                    annotation.surrounds_attribute = tag_value
                    self.unpairedtag_stack.append(annotation)
{'fraction_non_alphanumeric': 0.08437744714173845, 'fraction_numerical': 0.0011746280344557558, 'mean_word_length': 2.9662267080745344, 'pattern_counts': {'":': 3, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2130504', 'n_tokens_mistral': 3030, 'n_tokens_neox': 2904, 'n_words': 655}