<commit_before>"""
Parses search terms input by user

Searches take two primary forms:
- search term by itself (ex. "shh" or "brain")
- key / separator / value(s) (ex. "LRS=(9 99 Chr4 122 155)" or "GO:342533")

In the example of "LRS=(9 99 Chr4 122 155)", the key is "LRS", the separator is "=" and the value
is everything within the parentheses.

Both "=" and ":" can be used as separators; in the future, it would also be good to allow no
separator at all (ex. "cisLRS(9 999 10)")

Both square brackets and parentheses can be used interchangeably. Both can also be used to
encapsulate a single value; "cisLRS=[9 999 10)" would
be acceptable.]

"""

import re

from pprint import pformat as pf

from utility.logger import getLogger
logger = getLogger(__name__ )

def parse(pstring):
    """

    returned item search_term is always a list, even if only one element
    """
    pstring = re.split(r"""(?:(\w+\s*=\s*[\('"\[][^)'"]*[\)\]'"])  |  # LRS=(1 2 3), cisLRS=[4 5 6], etc
                       (\w+\s*[=:\>\<][\w\*]+)  |  # wiki=bar, GO:foobar, etc
                       (".*?") | ('.*?') | # terms in quotes, i.e. "brain weight"
                       ([\w\*]+))  # shh, brain, etc """, pstring,
                                                    flags=re.VERBOSE)

    pstring = [item.strip() for item in pstring if item and item.strip()]
    logger.debug("pstring:", pstring)

    items = []

    separators = [re.escape(x) for x in ("<=", ">=", ":", "=", "<", ">")]<commit_msg>Changed parser.py to account for the ? wildcard in the same way as GN1, but I suspect GN1 isn't actually doing this correctly (might have to do with switch to MariaDB)<commit_after>"""
Parses search terms input by user

Searches take two primary forms:
- search term by itself (ex. "shh" or "brain")
- key / separator / value(s) (ex. "LRS=(9 99 Chr4 122 155)" or "GO:342533")

In the example of "LRS=(9 99 Chr4 122 155)", the key is "LRS", the separator is "=" and the value
is everything within the parentheses.

Both "=" and ":" can be used as separators; in the future, it would also be good to allow no
separator at all (ex. "cisLRS(9 999 10)")

Both square brackets and parentheses can be used interchangeably. Both can also be used to
encapsulate a single value; "cisLRS=[9 999 10)" would
be acceptable.]

"""

import re

from pprint import pformat as pf

from utility.logger import getLogger
logger = getLogger(__name__ )

def parse(pstring):
    """

    returned item search_term is always a list, even if only one element
    """
    pstring = re.split(r"""(?:(\w+\s*=\s*[\('"\[][^)'"]*[\)\]'"])  |  # LRS=(1 2 3), cisLRS=[4 5 6], etc
                       (\w+\s*[=:\>\<][\w\*]+)  |  # wiki=bar, GO:foobar, etc
                       (".*?") | ('.*?') | # terms in quotes, i.e. "brain weight"
                       ([\w\*\?]+))  # shh, brain, etc """, pstring,
                                                    flags=re.VERBOSE)

    pstring = [item.strip() for item in pstring if item and item.strip()]

    items = []

    separators = [re.escape(x) for x in ("<=", ">=", ":", "=", "<", ">")]
{'fraction_non_alphanumeric': 0.16548387096774195, 'fraction_numerical': 0.02903225806451613, 'mean_word_length': 2.614219114219114, 'pattern_counts': {'":': 4, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6783484', 'n_tokens_mistral': 1118, 'n_tokens_neox': 987, 'n_words': 379}