<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals
import itertools
import logging
import re

from regparser.citations import Label, remove_citation_overlaps
from regparser.layer.key_terms import KeyTerms
from regparser.tree.depth import heuristics, rules, markers as mtypes
from regparser.tree.depth.derive import derive_depths
from regparser.tree.interpretation import merge_labels, text_to_labels
from regparser.tree.struct import Node, treeify
from regparser.tree.xml_parser import tree_utils


logger = logging.getLogger(__name__)
_marker_regex = re.compile(
    r'^\s*(' +                 # line start
    '([0-9]+)' +               # digits
    '|([ivxlcdm]+)' +          # roman
    '|([A-Z]+)' +              # upper
    '|(<E[^>]*>[0-9]+)' +      # emphasized digit
    r')\s*\..*', re.DOTALL)    # followed by a period and then anything


_marker_stars_regex = re.compile(
    r'^\s*(' +                 # line start
    '([0-9]+)' +               # digits
    '|([ivxlcdm]+)' +          # roman
    '|([A-Z]+)' +              # upper
    '|(<E[^>]*>[0-9]+)' +      # emphasized digit
    r')\s+\* \* \*\s*$', re.DOTALL)  # followed by stars


def get_first_interp_marker(text):
    match = _marker_regex.match(text)
    if match:
        marker = text[:text.find('.')].strip()      # up to dot
        if '<' in marker:
            marker += '</E>'
        return marker
    match = _marker_stars_regex.match(text)
    if match:
        return text[:text.find('*')].strip()        # up to star


_first_markers = [re.compile(r'[\.|,|;|:|\-|—]\s*(' + marker + ')\.')
                  for marker in ['i', 'A', '1']]


def collapsed_markers_matches(node_text, tagged_text):
    """Find collapsed markers, i.e. tree node paragraphs that begin within a
    single XML node, within this text. Remove citations and other false
    positives. This is pretty hacky right now -- it focuses on the plain
    text but takes cues from the tagged text. @todo: streamline logic"""
    # In addition to the regex above, keyterms are an acceptable prefix. We
    # therefore convert keyterms to satisfy the above regex
    node_for_keyterms = Node(node_text, node_type=Node.INTERP,
                             label=[get_first_interp_marker(node_text)])
    node_for_keyterms.tagged_text = tagged_text
    keyterm = KeyTerms.keyterm_in_node(node_for_keyterms)
    if keyterm:
        node_text = node_text.replace(keyterm, '.' * len(keyterm))

    collapsed_markers = []
    for marker in _first_markers:
        possible = ((m, m.start(), m.end())
                    for m in marker.finditer(node_text) if m.start() > 0)
        possible = remove_citation_overlaps(node_text, possible)
        # If certain characters follow, kill it
        for following in ("e.", ")", "”", '"', "'"):
            possible = [(m, s, end) for m, s, end in possible
                        if not node_text[end:].startswith(following)]
        possible = [m for m, _, _ in possible]
        # As all "1." collapsed markers must be emphasized, run a quick
        # check to weed out some false positives
        if '<E T="03">1' not in tagged_text:
            possible = filter(lambda m: m.group(1) != '1', possible)
        collapsed_markers.extend(possible)
    return collapsed_markers


def is_title(xml_node):
    """Not all titles are created equal. Sometimes a title appears as a
    paragraph tag, mostly to add confusion."""
    if xml_node.getchildren():
        child = xml_node.getchildren()[0]
    else:
        child = None
    return bool(
        (
            xml_node.tag.upper() == 'HD' and
            xml_node.attrib['SOURCE'] != 'HED') or
        (
            xml_node.tag.upper() == 'P' and
            (xml_node.text is None or not xml_node.text.strip()) and
            len(xml_node.getchildren()) == 1 and
            (child.tail is None or not child.tail.strip(" \n\t.")) and
            text_to_labels(child.text, Label(), warn=False)) or
        (
            xml_node.tag.upper() == 'P' and
            len(xml_node.getchildren()) == 0 and
            xml_node.text and not get_first_interp_marker(xml_node.text) and
            text_to_labels(xml_node.text, Label(), warn=False,
                           force_start=True)))


def process_inner_children(inner_stack, xml_node):
    """Process the following nodes as children of this interpretation. This
    is very similar to reg_text.py:build_from_section()"""
    children = itertools.takewhile(
        lambda x: not is_title(x), xml_node.itersiblings())
    nodes = []
    for xml_node in filter(lambda c: c.tag in ('P', 'STARS'), children):
        node_text = tree_utils.get_node_text(xml_node, add_spaces=True)
        text_with_tags = tree_utils.get_node_text_tags_preserved(xml_node)
        first_marker = get_first_interp_marker(text_with_tags)
        if xml_node.tag == 'STARS':
            nodes.append(Node(label=[mtypes.STARS_TAG]))
        elif not first_marker and nodes:
            logger.warning("Couldn't determine interp marker. Appending to "
                           "previous paragraph: %s", node_text)
            previous = nodes[-1]
            previous.text += "\n\n" + node_text
            if hasattr(previous, 'tagged_text'):
                previous.tagged_text += "\n\n" + text_with_tags
            else:
                previous.tagged_text = text_with_tags
        else:
            collapsed = collapsed_markers_matches(node_text, text_with_tags)

            #   -2 throughout to account for matching the character + period
            ends = [m.end() - 2 for m in collapsed[1:]] + [len(node_text)]
            starts = [m.end() - 2 for m in collapsed] + [len(node_text)]

            #   Node for this paragraph
            n = Node(node_text[0:starts[0]], label=[first_marker],
                     node_type=Node.INTERP)
            n.tagged_text = text_with_tags
            nodes.append(n)
            if n.text.endswith('* * *'):
                nodes.append(Node(label=[mtypes.INLINE_STARS]))

            #   Collapsed-marker children
            for match, end in zip(collapsed, ends):
                marker = match.group(1)
                if marker == '1':
                    marker = '<E T="03">1</E>'
                n = Node(node_text[match.end() - 2:end], label=[marker],
                         node_type=Node.INTERP)
                nodes.append(n)
                if n.text.endswith('* * *'):
                    nodes.append(Node(label=[mtypes.INLINE_STARS]))

    # Trailing stars don't matter; slightly more efficient to ignore them
    while nodes and nodes[-1].label[0] in mtypes.stars:
        nodes = nodes[:-1]

    # Use constraint programming to figure out possible depth assignments
    depths = derive_depths(
        [node.label[0] for node in nodes],<commit_msg>Split off a chunk of process_inner_children<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals
import itertools
import logging
import re

from regparser.citations import Label, remove_citation_overlaps
from regparser.layer.key_terms import KeyTerms
from regparser.tree.depth import heuristics, rules, markers as mtypes
from regparser.tree.depth.derive import derive_depths
from regparser.tree.interpretation import merge_labels, text_to_labels
from regparser.tree.struct import Node, treeify
from regparser.tree.xml_parser import tree_utils


logger = logging.getLogger(__name__)
_marker_regex = re.compile(
    r'^\s*(' +                 # line start
    '([0-9]+)' +               # digits
    '|([ivxlcdm]+)' +          # roman
    '|([A-Z]+)' +              # upper
    '|(<E[^>]*>[0-9]+)' +      # emphasized digit
    r')\s*\..*', re.DOTALL)    # followed by a period and then anything


_marker_stars_regex = re.compile(
    r'^\s*(' +                 # line start
    '([0-9]+)' +               # digits
    '|([ivxlcdm]+)' +          # roman
    '|([A-Z]+)' +              # upper
    '|(<E[^>]*>[0-9]+)' +      # emphasized digit
    r')\s+\* \* \*\s*$', re.DOTALL)  # followed by stars


def get_first_interp_marker(text):
    match = _marker_regex.match(text)
    if match:
        marker = text[:text.find('.')].strip()      # up to dot
        if '<' in marker:
            marker += '</E>'
        return marker
    match = _marker_stars_regex.match(text)
    if match:
        return text[:text.find('*')].strip()        # up to star


_first_markers = [re.compile(r'[\.|,|;|:|\-|—]\s*(' + marker + ')\.')
                  for marker in ['i', 'A', '1']]


def collapsed_markers_matches(node_text, tagged_text):
    """Find collapsed markers, i.e. tree node paragraphs that begin within a
    single XML node, within this text. Remove citations and other false
    positives. This is pretty hacky right now -- it focuses on the plain
    text but takes cues from the tagged text. @todo: streamline logic"""
    # In addition to the regex above, keyterms are an acceptable prefix. We
    # therefore convert keyterms to satisfy the above regex
    node_for_keyterms = Node(node_text, node_type=Node.INTERP,
                             label=[get_first_interp_marker(node_text)])
    node_for_keyterms.tagged_text = tagged_text
    keyterm = KeyTerms.keyterm_in_node(node_for_keyterms)
    if keyterm:
        node_text = node_text.replace(keyterm, '.' * len(keyterm))

    collapsed_markers = []
    for marker in _first_markers:
        possible = ((m, m.start(), m.end())
                    for m in marker.finditer(node_text) if m.start() > 0)
        possible = remove_citation_overlaps(node_text, possible)
        # If certain characters follow, kill it
        for following in ("e.", ")", "”", '"', "'"):
            possible = [(m, s, end) for m, s, end in possible
                        if not node_text[end:].startswith(following)]
        possible = [m for m, _, _ in possible]
        # As all "1." collapsed markers must be emphasized, run a quick
        # check to weed out some false positives
        if '<E T="03">1' not in tagged_text:
            possible = filter(lambda m: m.group(1) != '1', possible)
        collapsed_markers.extend(possible)
    return collapsed_markers


def is_title(xml_node):
    """Not all titles are created equal. Sometimes a title appears as a
    paragraph tag, mostly to add confusion."""
    if xml_node.getchildren():
        child = xml_node.getchildren()[0]
    else:
        child = None
    return bool(
        (
            xml_node.tag.upper() == 'HD' and
            xml_node.attrib['SOURCE'] != 'HED') or
        (
            xml_node.tag.upper() == 'P' and
            (xml_node.text is None or not xml_node.text.strip()) and
            len(xml_node.getchildren()) == 1 and
            (child.tail is None or not child.tail.strip(" \n\t.")) and
            text_to_labels(child.text, Label(), warn=False)) or
        (
            xml_node.tag.upper() == 'P' and
            len(xml_node.getchildren()) == 0 and
            xml_node.text and not get_first_interp_marker(xml_node.text) and
            text_to_labels(xml_node.text, Label(), warn=False,
                           force_start=True)))


def process_inner_children(inner_stack, xml_node):
    """Process the following nodes as children of this interpretation. This
    is very similar to reg_text.py:build_from_section()"""
    children = itertools.takewhile(
        lambda x: not is_title(x), xml_node.itersiblings())
    nodes = []
    for xml_node in filter(lambda c: c.tag in ('P', 'STARS'), children):
        node_text = tree_utils.get_node_text(xml_node, add_spaces=True)
        text_with_tags = tree_utils.get_node_text_tags_preserved(xml_node)
        first_marker = get_first_interp_marker(text_with_tags)
        if xml_node.tag == 'STARS':
            nodes.append(Node(label=[mtypes.STARS_TAG]))
        elif not first_marker and nodes:
            logger.warning("Couldn't determine interp marker. Appending to "
                           "previous paragraph: %s", node_text)
            previous = nodes[-1]
            previous.text += "\n\n" + node_text
            if hasattr(previous, 'tagged_text'):
                previous.tagged_text += "\n\n" + text_with_tags
            else:
                previous.tagged_text = text_with_tags
        else:
            collapsed = collapsed_markers_matches(node_text, text_with_tags)

            #   -2 throughout to account for matching the character + period
            ends = [m.end() - 2 for m in collapsed[1:]] + [len(node_text)]
            starts = [m.end() - 2 for m in collapsed] + [len(node_text)]

            #   Node for this paragraph
            n = Node(node_text[0:starts[0]], label=[first_marker],
                     node_type=Node.INTERP)
            n.tagged_text = text_with_tags
            nodes.append(n)
            if n.text.endswith('* * *'):
                nodes.append(Node(label=[mtypes.INLINE_STARS]))

            #   Collapsed-marker children
            for match, end in zip(collapsed, ends):
                marker = match.group(1)
                if marker == '1':
                    marker = '<E T="03">1</E>'
                n = Node(node_text[match.end() - 2:end], label=[marker],
                         node_type=Node.INTERP)
                nodes.append(n)
                if n.text.endswith('* * *'):
                    nodes.append(Node(label=[mtypes.INLINE_STARS]))

    # Trailing stars don't matter; slightly more efficient to ignore them
    while nodes and nodes[-1].label[0] in mtypes.stars:
        nodes = nodes[:-1]

    add_nodes_to_stack(nodes, inner_stack)


def add_nodes_to_stack(nodes, inner_stack):
    """Calculate most likely depth assignments to each node; add to the
    provided stack"""
    # Use constraint programming to figure out possible depth assignments
    depths = derive_depths(
        [node.label[0] for node in nodes],
{'fraction_non_alphanumeric': 0.12117282616526187, 'fraction_numerical': 0.005331028023917586, 'mean_word_length': 2.2802457466918713, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 42}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3303799', 'n_tokens_mistral': 4285, 'n_tokens_neox': 4134, 'n_words': 1110}