codingutf8 2022 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license feature extractor class for markuplm def initself kwargs requiresbackendsself bs4 super initkwargs def xpathsoupself element xpathtags xpathsubscripts child element if element name else element parent for parent in child parents type bs4 element tag siblings parent findallchild name recursivefalse xpathtags appendchild name xpathsubscripts append 0 if 1 lensiblings else nexti for i s in enumeratesiblings 1 if s is child child parent xpathtags reverse xpathsubscripts reverse return xpathtags xpathsubscripts def getthreefromsingleself htmlstring htmlcode beautifulsouphtmlstring html parser alldocstrings string2xtagseq string2xsubsseq for element in htmlcode descendants if isinstanceelement bs4 element navigablestring if typeelement parent bs4 element tag continue textinthistag html unescapeelement strip if not textinthistag continue alldocstrings appendtextinthistag xpathtags xpathsubscripts self xpathsoupelement string2xtagseq appendxpathtags string2xsubsseq appendxpathsubscripts if lenalldocstrings lenstring2xtagseq raise valueerrornumber of doc strings and xtags does not correspond if lenalldocstrings lenstring2xsubsseq raise valueerrornumber of doc strings and xsubs does not correspond return alldocstrings string2xtagseq string2xsubsseq def constructxpathself xpathtags xpathsubscripts xpath for tagname subs in zipxpathtags xpathsubscripts xpath ftagname if subs 0 xpath fsubs return xpath def callself htmlstrings batchfeature input type checking for clearer error check that strings has a valid type get nodes xpaths return as dict coding utf 8 2022 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license feature extractor class for markuplm constructs a markuplm feature extractor this can be used to get a list of nodes and corresponding xpaths from html strings this feature extractor inherits from feature_extraction_utils pretrainedfeatureextractor which contains most of the main methods users should refer to this superclass for more information regarding those methods type bs4 element tag main method to prepare for the model one or several html strings args html_strings str list str the html string or batch of html strings from which to extract nodes and corresponding xpaths returns batchfeature a batchfeature with the following fields nodes nodes xpaths corresponding xpaths examples python from transformers import markuplmfeatureextractor page_name_1 page1 html page_name_2 page2 html page_name_3 page3 html with open page_name_1 as f single_html_string f read feature_extractor markuplmfeatureextractor single example encoding feature_extractor single_html_string print encoding keys dict_keys nodes xpaths batched example multi_html_strings with open page_name_2 as f multi_html_strings append f read with open page_name_3 as f multi_html_strings append f read encoding feature_extractor multi_html_strings print encoding keys dict_keys nodes xpaths input type checking for clearer error check that strings has a valid type get nodes xpaths return as dict
import html

from ...feature_extraction_utils import BatchFeature, FeatureExtractionMixin
from ...utils import is_bs4_available, logging, requires_backends


if is_bs4_available():
    import bs4
    from bs4 import BeautifulSoup


logger = logging.get_logger(__name__)


class MarkupLMFeatureExtractor(FeatureExtractionMixin):
    r

    def __init__(self, **kwargs):
        requires_backends(self, ["bs4"])
        super().__init__(**kwargs)

    def xpath_soup(self, element):
        xpath_tags = []
        xpath_subscripts = []
        child = element if element.name else element.parent
        for parent in child.parents:  
            siblings = parent.find_all(child.name, recursive=False)
            xpath_tags.append(child.name)
            xpath_subscripts.append(
                0 if 1 == len(siblings) else next(i for i, s in enumerate(siblings, 1) if s is child)
            )
            child = parent
        xpath_tags.reverse()
        xpath_subscripts.reverse()
        return xpath_tags, xpath_subscripts

    def get_three_from_single(self, html_string):
        html_code = BeautifulSoup(html_string, "html.parser")

        all_doc_strings = []
        string2xtag_seq = []
        string2xsubs_seq = []

        for element in html_code.descendants:
            if isinstance(element, bs4.element.NavigableString):
                if type(element.parent) != bs4.element.Tag:
                    continue

                text_in_this_tag = html.unescape(element).strip()
                if not text_in_this_tag:
                    continue

                all_doc_strings.append(text_in_this_tag)

                xpath_tags, xpath_subscripts = self.xpath_soup(element)
                string2xtag_seq.append(xpath_tags)
                string2xsubs_seq.append(xpath_subscripts)

        if len(all_doc_strings) != len(string2xtag_seq):
            raise ValueError("Number of doc strings and xtags does not correspond")
        if len(all_doc_strings) != len(string2xsubs_seq):
            raise ValueError("Number of doc strings and xsubs does not correspond")

        return all_doc_strings, string2xtag_seq, string2xsubs_seq

    def construct_xpath(self, xpath_tags, xpath_subscripts):
        xpath = ""
        for tagname, subs in zip(xpath_tags, xpath_subscripts):
            xpath += f"/{tagname}"
            if subs != 0:
                xpath += f"[{subs}]"
        return xpath

    def __call__(self, html_strings) -> BatchFeature:
        

        
        valid_strings = False

        
        if isinstance(html_strings, str):
            valid_strings = True
        elif isinstance(html_strings, (list, tuple)):
            if len(html_strings) == 0 or isinstance(html_strings[0], str):
                valid_strings = True

        if not valid_strings:
            raise ValueError(
                "HTML strings must of type `str`, `List[str]` (batch of examples), "
                f"but is of type {type(html_strings)}."
            )

        is_batched = bool(isinstance(html_strings, (list, tuple)) and (isinstance(html_strings[0], str)))

        if not is_batched:
            html_strings = [html_strings]

        
        nodes = []
        xpaths = []
        for html_string in html_strings:
            all_doc_strings, string2xtag_seq, string2xsubs_seq = self.get_three_from_single(html_string)
            nodes.append(all_doc_strings)
            xpath_strings = []
            for node, tag_list, sub_list in zip(all_doc_strings, string2xtag_seq, string2xsubs_seq):
                xpath_string = self.construct_xpath(tag_list, sub_list)
                xpath_strings.append(xpath_string)
            xpaths.append(xpath_strings)

        
        data = {"nodes": nodes, "xpaths": xpaths}
        encoded_inputs = BatchFeature(data=data, tensor_type=None)

        return encoded_inputs