<commit_before># vim: set fileencoding=utf-8
from collections import defaultdict
import re

from inflection import pluralize

from regparser import utils
from regparser.grammar.external_citations import uscode_exp as uscode
from regparser.grammar.terms import term_parser, xml_term_parser
from regparser.layer.layer import Layer
from regparser.layer.paragraph_markers import ParagraphMarkers
from regparser.tree import struct
import settings


class Ref(object):
    def __init__(self, term, label, position):
        self.term = term
        self.label = label
        self.position = position

    def __eq__(self, other):
        """Equality depends on equality of the fields"""
        return (hasattr(other, 'term') and hasattr(other, 'label')
                and hasattr(other, 'position') and self.term == other.term
                and self.label == other.label
                and self.position == other.position)


class Terms(Layer):

    def __init__(self, tree):
        Layer.__init__(self, tree)
        self.layer['referenced'] = {}
        #   scope -> List[(term, definition_ref)]
        self.scoped_terms = defaultdict(list)
        #   subpart -> list[section]
        self.subpart_map = defaultdict(list)


    def add_subparts(self):
        """Document the relationship between sections and subparts"""

        current_subpart = [None]    # Need a reference for the closure

        def per_node(node):
            if node.node_type == struct.Node.SUBPART:
                current_subpart[0] = node.label[2]
            elif node.node_type == struct.Node.EMPTYPART:
                current_subpart[0] = None
            if (len(node.label) == 2 and
                node.node_type in (struct.Node.REGTEXT, struct.Node.APPENDIX)):
                #Subparts
                section = node.label[-1]
                self.subpart_map[current_subpart[0]].append(section)

        struct.walk(self.tree, per_node)

    def pre_process(self):
        """Step through every node in the tree, finding definitions. Add
        these definition to self.scoped_terms. Also keep track of which
        subpart we are in. Finally, document all defined terms. """

        self.add_subparts()

        def per_node(node):
            if self.has_definitions(node):
                for scope in self.definitions_scopes(node):
                    included, excluded = self.node_definitions(node)
                    self.scoped_terms[scope].extend(included)
                    self.scoped_terms['EXCLUDED'].extend(excluded)

        struct.walk(self.tree, per_node)

        for scope in self.scoped_terms:
            for ref in self.scoped_terms[scope]:
                self.layer['referenced'][ref.term + ":" + ref.label] = {
                    'term': ref.term,
                    'reference': ref.label,
                    'position': ref.position
                }

    def has_definitions(self, node):
        """Does this node have definitions?"""
        # Definitions cannot be in the top-most layer of the tree (the root)
        if len(node.label) < 2:
            return False
        # Definitions are only in the reg text (not appendices/interprs)
        if node.node_type != struct.Node.REGTEXT:
            return False
        stripped = node.text.strip(ParagraphMarkers.marker(node)).strip()
        return (
            stripped.lower().startswith('definition')
            or (node.title and 'definition' in node.title.lower())
            or re.search('the term .* means', stripped.lower())
            )

    def is_exclusion(self, term, text, previous_terms):
        """Some definitions are exceptions/exclusions of a previously
        defined term. At the moment, we do not want to include these as they
        would replace previous (correct) definitions."""
        if term not in [t.term for t in previous_terms]:
            return False
        regex = 'the term .?' + re.escape(term) + '.? does not include'
        return bool(re.search(regex, text.lower()))

    def node_definitions(self, node):
        """Walk through this node and its children to find defined terms.
        'Act' is a special case, as it is also defined as an external
        citation."""
        included_defs = []
        excluded_defs = []

        def add_match(n, match):
            term = match.term.tokens[0].lower()
            pos = match.term.pos

            add_to = included_defs

            if term == 'act' and list(uscode.scanString(n.text)):
                add_to = excluded_defs
            if self.is_exclusion(term, n.text, included_defs):
                add_to = excluded_defs
            add_to.append(Ref(term, n.label_id(), pos))

        def per_node(n):
            for match, _, _ in term_parser.scanString(n.text):
                add_match(n, match)

            if hasattr(n, 'tagged_text'):
                for match, _, _ in xml_term_parser.scanString(n.tagged_text):
                    """Returned position tuple has positions relating
                    to n.tagged_text. Replace with positions that apply
                    to n.text."""
                    pos_start = n.text.find(match.term.tokens[0])
                    match_len = len(match.term.tokens[0])
                    match.term.pos = (pos_start, pos_start + match_len)
                    add_match(n, match)

        struct.walk(node, per_node)
        return included_defs, excluded_defs

    def subpart_scope(self, label_parts):
        """Given a label, determine which sections fall under the same
        subpart"""
        reg = label_parts[0]
        section = label_parts[1]
        for subpart in self.subpart_map:
            if section in self.subpart_map[subpart]:
                return [[reg, sect] for sect in self.subpart_map[subpart]]
        return []

    def definitions_scopes(self, node):
        """Try to determine the scope of definitions in this term."""
        scopes = []
        if "purposes of this part" in node.text.lower():
            scopes.append(node.label[:1])
        elif "purposes of this subpart" in node.text.lower():
            scopes.extend(self.subpart_scope(node.label))
        elif "purposes of this section" in node.text.lower():
            scopes.append(node.label[:2])
        elif "purposes of this paragraph" in node.text.lower():
            scopes.append(node.label)
        else:   # defaults to whole reg
            scopes.append(node.label[:1])
<commit_msg>pass term and position to add_match explicitly<commit_after># vim: set fileencoding=utf-8
from collections import defaultdict
import re

from inflection import pluralize

from regparser import utils
from regparser.grammar.external_citations import uscode_exp as uscode
from regparser.grammar.terms import term_parser, xml_term_parser
from regparser.layer.layer import Layer
from regparser.layer.paragraph_markers import ParagraphMarkers
from regparser.tree import struct
import settings


class Ref(object):
    def __init__(self, term, label, position):
        self.term = term
        self.label = label
        self.position = position

    def __eq__(self, other):
        """Equality depends on equality of the fields"""
        return (hasattr(other, 'term') and hasattr(other, 'label')
                and hasattr(other, 'position') and self.term == other.term
                and self.label == other.label
                and self.position == other.position)


class Terms(Layer):

    def __init__(self, tree):
        Layer.__init__(self, tree)
        self.layer['referenced'] = {}
        #   scope -> List[(term, definition_ref)]
        self.scoped_terms = defaultdict(list)
        #   subpart -> list[section]
        self.subpart_map = defaultdict(list)


    def add_subparts(self):
        """Document the relationship between sections and subparts"""

        current_subpart = [None]    # Need a reference for the closure

        def per_node(node):
            if node.node_type == struct.Node.SUBPART:
                current_subpart[0] = node.label[2]
            elif node.node_type == struct.Node.EMPTYPART:
                current_subpart[0] = None
            if (len(node.label) == 2 and
                node.node_type in (struct.Node.REGTEXT, struct.Node.APPENDIX)):
                #Subparts
                section = node.label[-1]
                self.subpart_map[current_subpart[0]].append(section)

        struct.walk(self.tree, per_node)

    def pre_process(self):
        """Step through every node in the tree, finding definitions. Add
        these definition to self.scoped_terms. Also keep track of which
        subpart we are in. Finally, document all defined terms. """

        self.add_subparts()

        def per_node(node):
            if self.has_definitions(node):
                for scope in self.definitions_scopes(node):
                    included, excluded = self.node_definitions(node)
                    self.scoped_terms[scope].extend(included)
                    self.scoped_terms['EXCLUDED'].extend(excluded)

        struct.walk(self.tree, per_node)

        for scope in self.scoped_terms:
            for ref in self.scoped_terms[scope]:
                self.layer['referenced'][ref.term + ":" + ref.label] = {
                    'term': ref.term,
                    'reference': ref.label,
                    'position': ref.position
                }

    def has_definitions(self, node):
        """Does this node have definitions?"""
        # Definitions cannot be in the top-most layer of the tree (the root)
        if len(node.label) < 2:
            return False
        # Definitions are only in the reg text (not appendices/interprs)
        if node.node_type != struct.Node.REGTEXT:
            return False
        stripped = node.text.strip(ParagraphMarkers.marker(node)).strip()
        return (
            stripped.lower().startswith('definition')
            or (node.title and 'definition' in node.title.lower())
            or re.search('the term .* means', stripped.lower())
            )

    def is_exclusion(self, term, text, previous_terms):
        """Some definitions are exceptions/exclusions of a previously
        defined term. At the moment, we do not want to include these as they
        would replace previous (correct) definitions."""
        if term not in [t.term for t in previous_terms]:
            return False
        regex = 'the term .?' + re.escape(term) + '.? does not include'
        return bool(re.search(regex, text.lower()))

    def node_definitions(self, node):
        """Walk through this node and its children to find defined terms.
        'Act' is a special case, as it is also defined as an external
        citation."""
        included_defs = []
        excluded_defs = []

        def add_match(n, term, pos):
            add_to = included_defs

            if term == 'act' and list(uscode.scanString(n.text)):
                add_to = excluded_defs
            if self.is_exclusion(term, n.text, included_defs):
                add_to = excluded_defs
            add_to.append(Ref(term, n.label_id(), pos))

        def per_node(n):
            for match, _, _ in term_parser.scanString(n.text):
                add_match(n,
                          match.term.tokens[0].lower(),
                          match.term.pos)

            if hasattr(n, 'tagged_text'):
                for match, _, _ in xml_term_parser.scanString(n.tagged_text):
                    """Position in match reflects XML tags, so its
                    dropped in preference of new values based on
                    n.text."""
                    pos_start = n.text.find(match.term.tokens[0])
                    match_len = len(match.term.tokens[0])
                    add_match(n,
                              match.term.tokens[0].lower(),
                              (pos_start, pos_start + match_len))

        struct.walk(node, per_node)
        return included_defs, excluded_defs

    def subpart_scope(self, label_parts):
        """Given a label, determine which sections fall under the same
        subpart"""
        reg = label_parts[0]
        section = label_parts[1]
        for subpart in self.subpart_map:
            if section in self.subpart_map[subpart]:
                return [[reg, sect] for sect in self.subpart_map[subpart]]
        return []

    def definitions_scopes(self, node):
        """Try to determine the scope of definitions in this term."""
        scopes = []
        if "purposes of this part" in node.text.lower():
            scopes.append(node.label[:1])
        elif "purposes of this subpart" in node.text.lower():
            scopes.extend(self.subpart_scope(node.label))
        elif "purposes of this section" in node.text.lower():
            scopes.append(node.label[:2])
        elif "purposes of this paragraph" in node.text.lower():
            scopes.append(node.label)
        else:   # defaults to whole reg
            scopes.append(node.label[:1])

{'fraction_non_alphanumeric': 0.09784283513097072, 'fraction_numerical': 0.002542372881355932, 'mean_word_length': 2.0550717815956694, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3709720', 'n_tokens_mistral': 3633, 'n_tokens_neox': 3497, 'n_words': 1003}