<commit_before>from lxml import etree
import StringIO

import re


LEFT_WHITE_SPACE_RE = re.compile('^[ \n\t].*$')
RIGHT_WHITE_SPACE_RE = re.compile('^.*[ \n\t]$')


def strip_text(text):
    """
    Takes a bunch of test and removes all possible "indentation gunk"
    from it.

      >>> example_noisy_text = '''
      ...  hey guys
      ...       it looks like 
      ... I am all over the place'''
      >>> strip_text(example_noisy_text)
      u'hey guys it looks like I am all over the place'
    """
    if text:
        return u' '.join(
            [line.strip() for line in text.splitlines() if line.strip()])
    else:
        # return whatever text was, there's nothing to do
        # (prolly None or empty string)
        return text


def strip_xml(element):
    """
    Recursively strip clean indentation from xml.  Especially useful
    if you're using a template.

    For example, this is a bit of a mess:
      >>> xml_mess = '''
      ... <help> How did
      ... 
      ...  <person>I
      ...    </person>
      ... get to be so <cleanliness
      ...     xmlns:clean="http://example.org/howclean/#"
      ...     clean:cleanliness="filthy">messy</cleanliness>?
      ...    </help>  '''

    strip_xml requires that you pass in an element though, so let's
    get the root node and pass it in:

      >>> from lxml import etree
      >>> import StringIO
      >>> etree_mess = etree.parse(StringIO.StringIO(xml_mess))
      >>> cleaned_root_mess = strip_xml(etree_mess.getroot())
      >>> etree.tostring(cleaned_root_mess)
      '<help>How did <person>I</person> get to be so <cleanliness xmlns:clean="http://example.org/howclean/#" clean:cleanliness="filthy">messy</cleanliness>?</help>'

    Note that strip_xml operates on the mutability of the argument
    `element`, so the object returned is the same object that's passed
    in.

      >>> cleaned_root_mess is etree_mess.getroot()
      True
    """
    def _recursive_strip(elt, childpos, childrenlen):
        orig_text = elt.text or ''
        orig_tail = elt.tail or ''

        children = list(elt)
        new_childrenlen = len(children)

        # We have to do a lot of stuff here to put whitespace in the
        # right places and make it look pretty, as if a human wrote
        # it.
        split_orig_text = orig_text.splitlines()
        orig_text_butt = None
        if split_orig_text:
            orig_text_butt = split_orig_text[-1]

        split_orig_tail = orig_tail.splitlines()
        orig_tail_butt = None
        if split_orig_tail:
            orig_tail_butt = split_orig_tail[-1]

        elt.text = strip_text(elt.text)
        elt.tail = strip_text(elt.tail)

        ##########
        #### whitespace re-appending
        ##########

        ####
        ## left of the .text
        ####
        # pretty much never

        ####
        ## right of the .text
        ####
        # if there are children and is presently whitespace
        if orig_text_butt \
                and new_childrenlen \
                and RIGHT_WHITE_SPACE_RE.match(orig_text_butt):
            elt.text = elt.text + ' '

        ####
        ## left of the .tail
        ####
        # any time there is presently whitespace
        if LEFT_WHITE_SPACE_RE.match(orig_tail):
            elt.tail = ' ' + elt.tail

        ####
        ## right of the .tail
        ####
        # if there is presently whitespace and not the last child
        if orig_tail_butt \
                and RIGHT_WHITE_SPACE_RE.match(orig_tail_butt) \
                and childpos != childrenlen - 1:
            elt.tail = elt.tail + ' '

        for i in range(new_childrenlen):
            child = children[i]
            _recursive_strip(child, i, new_childrenlen)

    _recursive_strip(element, 0, 1)
    return element


def inner_xml(elt):
    """
    Get the inner xml of an element.

      >>> html_etree = etree.parse(
      ...   StringIO.StringIO(
      ...     '<div>This is some <i><b>really</b> silly</i> text!</div>'))
      >>> inner_xml(html_etree.getroot())
      u'This is some <i><b>really</b> silly</i> text!'<commit_msg>Don't add whitespace when there really should be nothing.<commit_after>from lxml import etree
import StringIO

import re


LEFT_WHITE_SPACE_RE = re.compile('^[ \n\t].*$')
RIGHT_WHITE_SPACE_RE = re.compile('^.*[ \n\t]$')


def strip_text(text):
    """
    Takes a bunch of test and removes all possible "indentation gunk"
    from it.

      >>> example_noisy_text = '''
      ...  hey guys
      ...       it looks like 
      ... I am all over the place'''
      >>> strip_text(example_noisy_text)
      u'hey guys it looks like I am all over the place'
    """
    if text:
        return u' '.join(
            [line.strip() for line in text.splitlines() if line.strip()])
    else:
        # return whatever text was, there's nothing to do
        # (prolly None or empty string)
        return text


def strip_xml(element):
    """
    Recursively strip clean indentation from xml.  Especially useful
    if you're using a template.

    For example, this is a bit of a mess:
      >>> xml_mess = '''
      ... <help> How did
      ... 
      ...  <person>I
      ...    </person>
      ... get to be so <cleanliness
      ...     xmlns:clean="http://example.org/howclean/#"
      ...     clean:cleanliness="filthy">messy</cleanliness>?
      ...    </help>  '''

    strip_xml requires that you pass in an element though, so let's
    get the root node and pass it in:

      >>> from lxml import etree
      >>> import StringIO
      >>> etree_mess = etree.parse(StringIO.StringIO(xml_mess))
      >>> cleaned_root_mess = strip_xml(etree_mess.getroot())
      >>> etree.tostring(cleaned_root_mess)
      '<help>How did <person>I</person> get to be so <cleanliness xmlns:clean="http://example.org/howclean/#" clean:cleanliness="filthy">messy</cleanliness>?</help>'

    Note that strip_xml operates on the mutability of the argument
    `element`, so the object returned is the same object that's passed
    in.

      >>> cleaned_root_mess is etree_mess.getroot()
      True
    """
    def _recursive_strip(elt, childpos, childrenlen):
        orig_text = elt.text or ''
        orig_tail = elt.tail or ''

        children = list(elt)
        new_childrenlen = len(children)

        # We have to do a lot of stuff here to put whitespace in the
        # right places and make it look pretty, as if a human wrote
        # it.
        split_orig_text = orig_text.splitlines()
        orig_text_butt = None
        if split_orig_text:
            orig_text_butt = split_orig_text[-1]

        split_orig_tail = orig_tail.splitlines()
        orig_tail_butt = None
        if split_orig_tail:
            orig_tail_butt = split_orig_tail[-1]

        elt.text = strip_text(elt.text)
        elt.tail = strip_text(elt.tail)

        ##########
        #### whitespace re-appending
        ##########

        ####
        ## left of the .text
        ####
        # pretty much never

        ####
        ## right of the .text
        ####
        # if there are children and is presently whitespace
        if orig_text_butt \
                and new_childrenlen \
                and RIGHT_WHITE_SPACE_RE.match(orig_text_butt):
            elt.text = elt.text + ' '

        ####
        ## left of the .tail
        ####
        # any time there is presently whitespace
        if orig_tail and LEFT_WHITE_SPACE_RE.match(orig_tail):
            elt.tail = ' ' + elt.tail

        ####
        ## right of the .tail
        ####
        # if there is presently whitespace and not the last child
        if orig_tail_butt \
                and RIGHT_WHITE_SPACE_RE.match(orig_tail_butt) \
                and childpos != childrenlen - 1:
            elt.tail = elt.tail + ' '

        for i in range(new_childrenlen):
            child = children[i]
            _recursive_strip(child, i, new_childrenlen)

    _recursive_strip(element, 0, 1)
    return element


def inner_xml(elt):
    """
    Get the inner xml of an element.

      >>> html_etree = etree.parse(
      ...   StringIO.StringIO(
      ...     '<div>This is some <i><b>really</b> silly</i> text!</div>'))
      >>> inner_xml(html_etree.getroot())
      u'This is some <i><b>really</b> silly</i> text!'
{'fraction_non_alphanumeric': 0.11834176910098819, 'fraction_numerical': 0.001205109664979513, 'mean_word_length': 2.017818181818182, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 113, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 28}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4074308', 'n_tokens_mistral': 2642, 'n_tokens_neox': 2514, 'n_words': 713}