<commit_before># vim: set fileencoding=utf-8
"""Parsers for finding a term that's being defined within a node"""
from itertools import chain
import re

from regparser.citations import Label
from regparser.grammar import terms as grammar
import settings


class Ref(object):
    """A reference to a defined term. Keeps track of the term, where it was
    found and the term's position in that node's text"""
    def __init__(self, term, label, start):
        self.term = unicode(term).lower()
        self.label = label
        self.start = start
        self.end = self.start + len(term)
        self.position = (self.start, self.end)

    def __eq__(self, other):
        """Equality depends on equality of the fields"""
        return isinstance(other, Ref) and repr(self) == repr(other)

    def __repr__(self):
        return 'Ref( term=%s, label=%s, start=%s )' % (
            repr(self.term), repr(self.label), repr(self.start))


class ExplicitIncludes(object):
    """Definitions can be explicitly included in the settings. For example,
    say that a paragraph doesn't indicate that a certain phrase is a
    definition; we can define INCLUDE_DEFINITIONS_IN in our settings file,
    which will be checked here."""
    def find(self, node):
        refs = []
        cfr_part = node.label[0] if node.label else None
        for included_term, context in settings.INCLUDE_DEFINITIONS_IN.get(
                cfr_part, []):
            if context in node.text and included_term in node.text:
                pos_start = node.text.index(included_term)
                refs.append(Ref(included_term, node.label_id(), pos_start))
        return refs


class SmartQuotes(object):
    """Definitions indicated via smart quotes"""
    def __init__(self, stack):
        """Stack (which references ancestors of a node) is used to determine
        whether or not to apply smart quotes"""
        self.stack = stack

    def find(self, node):
        refs = []
        if self.stack and self.has_def_indicator():
            for match, _, _ in grammar.smart_quotes.scanString(node.text):
                term = match.term.tokens[0].lower().strip(',.;')
                refs.append(Ref(term, node.label_id(), match.term.pos[0]))
        return refs

    def has_def_indicator(self):
        """With smart quotes, we catch some false positives, phrases in quotes
        that are not terms. This extra test lets us know that a parent of the
        node looks like it would contain definitions."""
        for node in self.stack.lineage():
            lower_text = node.text.lower()
            in_text = 'Definition' in node.text
            in_title = 'Definition' in (node.title or '')
            pattern1 = re.search('the term .* (means|refers to)', lower_text)
            pattern2 = re.search(u'“[^”]+” (means|refers to)', lower_text)
            if in_text or in_title or pattern1 or pattern2:
                return True
        return False


class ScopeMatch(object):
    """We know these will be definitions because the scope of the definition
    is spelled out. E.g. 'for the purposes of XXX, the term YYY means'"""
    def __init__(self, finder):
        """Finder is an instance of ScopeFinder"""
        self.finder = finder

    def find(self, node):
        refs = []
        for match, _, _ in grammar.scope_term_type_parser.scanString(
                node.text):
            valid_scope = self.finder.scope_of_text(
                match.scope, Label.from_node(node), verify_prefix=False)
            valid_term = re.match("^[a-z ]+$", match.term.tokens[0])
            if valid_scope and valid_term:
                term = match.term.tokens[0].strip()
                pos_start = node.text.index(term, match.term.pos[0])
                refs.append(Ref(term, node.label_id(), pos_start))
        return refs


class XMLTermMeans(object):
    """Namespace for a matcher for e.g. '<E>XXX</E> means YYY'"""
    def __init__(self, existing_refs=None):
        """Existing refs will be used to exclude certain matches"""
        if existing_refs is None:
            existing_refs = []
        self.exclusions = list(existing_refs)<commit_msg>Use an abstract base class for definition finders<commit_after># vim: set fileencoding=utf-8
"""Parsers for finding a term that's being defined within a node"""
import abc
from itertools import chain
import re

from regparser.citations import Label
from regparser.grammar import terms as grammar
import settings


class Ref(object):
    """A reference to a defined term. Keeps track of the term, where it was
    found and the term's position in that node's text"""
    def __init__(self, term, label, start):
        self.term = unicode(term).lower()
        self.label = label
        self.start = start
        self.end = self.start + len(term)
        self.position = (self.start, self.end)

    def __eq__(self, other):
        """Equality depends on equality of the fields"""
        return isinstance(other, Ref) and repr(self) == repr(other)

    def __repr__(self):
        return 'Ref( term=%s, label=%s, start=%s )' % (
            repr(self.term), repr(self.label), repr(self.start))


class FinderBase(object):
    """Base class for all of the definition finder classes. Defines the
    interface they must implement"""
    __metaclass__ = abc.ABCMeta

    @abc.abstractmethod
    def find(self, node):
        """Given a Node, pull out any definitions it may contain as a list of
        Refs"""
        raise NotImplementedError()


class ExplicitIncludes(FinderBase):
    """Definitions can be explicitly included in the settings. For example,
    say that a paragraph doesn't indicate that a certain phrase is a
    definition; we can define INCLUDE_DEFINITIONS_IN in our settings file,
    which will be checked here."""
    def find(self, node):
        refs = []
        cfr_part = node.label[0] if node.label else None
        for included_term, context in settings.INCLUDE_DEFINITIONS_IN.get(
                cfr_part, []):
            if context in node.text and included_term in node.text:
                pos_start = node.text.index(included_term)
                refs.append(Ref(included_term, node.label_id(), pos_start))
        return refs


class SmartQuotes(FinderBase):
    """Definitions indicated via smart quotes"""
    def __init__(self, stack):
        """Stack (which references ancestors of a node) is used to determine
        whether or not to apply smart quotes"""
        self.stack = stack

    def find(self, node):
        refs = []
        if self.stack and self.has_def_indicator():
            for match, _, _ in grammar.smart_quotes.scanString(node.text):
                term = match.term.tokens[0].lower().strip(',.;')
                refs.append(Ref(term, node.label_id(), match.term.pos[0]))
        return refs

    def has_def_indicator(self):
        """With smart quotes, we catch some false positives, phrases in quotes
        that are not terms. This extra test lets us know that a parent of the
        node looks like it would contain definitions."""
        for node in self.stack.lineage():
            lower_text = node.text.lower()
            in_text = 'Definition' in node.text
            in_title = 'Definition' in (node.title or '')
            pattern1 = re.search('the term .* (means|refers to)', lower_text)
            pattern2 = re.search(u'“[^”]+” (means|refers to)', lower_text)
            if in_text or in_title or pattern1 or pattern2:
                return True
        return False


class ScopeMatch(FinderBase):
    """We know these will be definitions because the scope of the definition
    is spelled out. E.g. 'for the purposes of XXX, the term YYY means'"""
    def __init__(self, finder):
        """Finder is an instance of ScopeFinder"""
        self.finder = finder

    def find(self, node):
        refs = []
        for match, _, _ in grammar.scope_term_type_parser.scanString(
                node.text):
            valid_scope = self.finder.scope_of_text(
                match.scope, Label.from_node(node), verify_prefix=False)
            valid_term = re.match("^[a-z ]+$", match.term.tokens[0])
            if valid_scope and valid_term:
                term = match.term.tokens[0].strip()
                pos_start = node.text.index(term, match.term.pos[0])
                refs.append(Ref(term, node.label_id(), pos_start))
        return refs


class XMLTermMeans(FinderBase):
    """Namespace for a matcher for e.g. '<E>XXX</E> means YYY'"""
    def __init__(self, existing_refs=None):
        """Existing refs will be used to exclude certain matches"""
        if existing_refs is None:
            existing_refs = []
        self.exclusions = list(existing_refs)
{'fraction_non_alphanumeric': 0.10294286041451964, 'fraction_numerical': 0.0025191801213786786, 'mean_word_length': 2.5231948366276726, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7034120', 'n_tokens_mistral': 2542, 'n_tokens_neox': 2418, 'n_words': 830}