<commit_before>"""An extractor of boolean trigram queries from a regex, such that a superset
of the docs matching the regex are returned. You can then run the actual regex
over just the returned docs, saving a lot of computation and IO.

<NAME> and <NAME>, in "A fast regular expression indexing
engine", descibe an intuitive method for accelerating regex searching with a
trigram index. This is roughly an implementation of that.

<NAME>, in http://swtch.com/~rsc/regexp/regexp4.html, refines
that to {(1) extract use from runs of less than 3 static chars and (2) extract
trigrams that cross the boundaries between subexpressions} by keeping track of
prefix and suffix information while chewing through a pattern and effectively
merging adjacent subpatterns. This is a direction we may go in the future.

"""
from parsimonious import Grammar
from parsimonious.nodes import NodeVisitor


class RegexSummary(object):
    """The digested result of analyzing a parsed regex

    :attr can_match_empty: Whether the regex can match the empty string
    :attr exacts: Set of exact strings which, unioned, exhaust the regex. For
        example (s?printf) would yield {sprintf, printf}.
    :attr prefixes: The set of prefixes of strings the regex can match
    :attr suffixes: The set of suffixes of strings the regex can match
    :attr query: A TrigramTree that must be satisfied by any matching
        string, in addition to the restrictions expressed by the other
        attributes

    Prefixes, suffixes, and the rest are used only as intermediate values. The
    point is for them ultimately to become part of the query, which is itself a
    boolean combination of trigrams.

    """
    def __init__(self, regex):
        """Dispatch on the opcode of regex, and descend recursively, analyzing
        lower nodes and then pulling back up to finally summarize the whole.

        :arg regex: A parsed regex, as returned by ``sre_parse.parse()``

        """
        self.can_match_empty = can_match_empty
        self.exacts = exacts
        self.prefix = prefix  # This can probably be an actual set. The Go impl blows a lot of code removing dupes and such.
        self.suffix = suffix
        self.query = query


def summarize_regex(regex):
    """Return a RegexSummary of a regex.

    :arg regex: A string containing a regex pattern

    """


def trigram_query(regex):
    """Return an iterable of trigrams that will be found in any text matching
    the given regex.

    :arg regex: A string containing a regex pattern

    """
    # TODO: Veto patterns which are easy DOSes.

    # I suspect simplify(force=True) mashes everything down into `match` in
    # preparation for actually running `match` against a corpus.

    summary = RegexSummary(regex)
    summary.simplify(force=True)
    summary.add_exact()
    return summary.query


def trigrammify_tree(tree):
    """Explode the strings in a tree into trigrams. Return a tree whose leaves are trigrams and whose inner nodes are boolean operations. This can then be trivially turned into an ES query."""
    # Remember that Lucene regexes are always anchored. The caller will have to add a leading .*, etc.
    # Raise an exception somewhere, maybe in simplify_tree, if there are no trigrams.


def simplify_tree(tree):
    """Do the node reduction to collapse branches of a string tree that aren't
    useful. Remove strings that aren't long enough to generate any trigrams."""


# We should parse a regex. Then go over the tree and turn things like c+ into cc*, perhaps, as it makes it easier to see trigrams to extract.
# TODO: Parse normal regex syntax, but spit out Lucene-compatible syntax, with " escaped. And all special chars escaped even in character classes, in accordance with https://lucene.apache.org/core/4_6_0/core/org/apache/lucene/util/automaton/RegExp.html?is-external=true.

# TODO: Expand positive char classes so we can get trigrams out of [sp][rne]
# (trilite expands char classes of up to 10 chars but does nothing for larger<commit_msg>Remove some functions we're not going to need anymore.<commit_after>"""An extractor of boolean trigram queries from a regex, such that a superset
of the docs matching the regex are returned. You can then run the actual regex
over just the returned docs, saving a lot of computation and IO.

<NAME> and <NAME>, in "A fast regular expression indexing
engine", descibe an intuitive method for accelerating regex searching with a
trigram index. This is roughly an implementation of that.

<NAME>, in http://swtch.com/~rsc/regexp/regexp4.html, refines
that to {(1) extract use from runs of less than 3 static chars and (2) extract
trigrams that cross the boundaries between subexpressions} by keeping track of
prefix and suffix information while chewing through a pattern and effectively
merging adjacent subpatterns. This is a direction we may go in the future.

"""
from parsimonious import Grammar
from parsimonious.nodes import NodeVisitor


class RegexSummary(object):
    """The digested result of analyzing a parsed regex

    :attr can_match_empty: Whether the regex can match the empty string
    :attr exacts: Set of exact strings which, unioned, exhaust the regex. For
        example (s?printf) would yield {sprintf, printf}.
    :attr prefixes: The set of prefixes of strings the regex can match
    :attr suffixes: The set of suffixes of strings the regex can match
    :attr query: A TrigramTree that must be satisfied by any matching
        string, in addition to the restrictions expressed by the other
        attributes

    Prefixes, suffixes, and the rest are used only as intermediate values. The
    point is for them ultimately to become part of the query, which is itself a
    boolean combination of trigrams.

    """
    def __init__(self, regex):
        """Dispatch on the opcode of regex, and descend recursively, analyzing
        lower nodes and then pulling back up to finally summarize the whole.

        :arg regex: A parsed regex, as returned by ``sre_parse.parse()``

        """
        self.can_match_empty = can_match_empty
        self.exacts = exacts
        self.prefix = prefix  # This can probably be an actual set. The Go impl blows a lot of code removing dupes and such.
        self.suffix = suffix
        self.query = query


def summarize_regex(regex):
    """Return a RegexSummary of a regex.

    :arg regex: A string containing a regex pattern

    """


def trigram_query(regex):
    """Return an iterable of trigrams that will be found in any text matching
    the given regex.

    :arg regex: A string containing a regex pattern

    """
    # TODO: Veto patterns which are easy DOSes.

    # I suspect simplify(force=True) mashes everything down into `match` in
    # preparation for actually running `match` against a corpus.

    summary = RegexSummary(regex)
    summary.simplify(force=True)
    summary.add_exact()
    return summary.query


# We should parse a regex. Then go over the tree and turn things like c+ into cc*, perhaps, as it makes it easier to see trigrams to extract.
# TODO: Parse normal regex syntax, but spit out Lucene-compatible syntax, with " escaped. And all special chars escaped even in character classes, in accordance with https://lucene.apache.org/core/4_6_0/core/org/apache/lucene/util/automaton/RegExp.html?is-external=true.

# TODO: Expand positive char classes so we can get trigrams out of [sp][rne]
# (trilite expands char classes of up to 10 chars but does nothing for larger
{'fraction_non_alphanumeric': 0.059874649953327114, 'fraction_numerical': 0.002400320042672356, 'mean_word_length': 3.562043795620438, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6464913', 'n_tokens_mistral': 2037, 'n_tokens_neox': 1898, 'n_words': 1019}