<commit_before>"""
Parses search terms input by user

Searches take two primary forms:
- search term by itself (ex. "shh" or "brain")
- key / separator / value(s) (ex. "LRS=(9 99 Chr4 122 155)" or "GO:342533")

In the example of "LRS=(9 99 Chr4 122 155)", the key is "LRS", the separator is "=" and the value
is everything within the parentheses.

Both "=" and ":" can be used as separators; in the future, it would also be good to allow no
separator at all (ex. "cisLRS(9 999 10)")

Both square brackets and parentheses can be used interchangeably. Both can also be used to
encapsulate a single value; "cisLRS=[9 999 10)" would
be acceptable.]

NEED TO DEAL WITH WILDCARD CHARACTER '*'

"""

from __future__ import print_function, division

import re

from pprint import pformat as pf

def parse(pstring):
    pstring = re.split(r"""(?:(\w+\s*=\s*[\(\[][^)]*[\)\]])  |  # LRS=(1 2 3), cisLRS=[4 5 6], etc
                       (\w+\s*[=:]\w+)  |  # wiki=bar, GO:foobar, etc
                       (\w+))  # shh, brain, etc """, pstring,
                                                    flags=re.VERBOSE)
    pstring = [item.strip() for item in pstring if item and item.strip()]
    print(pstring)

    items = []

    for item in pstring:
        if ":" in item:
            key, seperator, value = item.partition(':')
        elif "=" in item:
            key, seperator, value = item.partition('=')
        else:
            seperator = None

        if seperator:
            if '(' in value or '[' in value:
                assert value.startswith(("(", "[")), "Invalid token"
                assert value.endswith((")", "]")), "Invalid token"
                value = value[1:-1] # Get rid of the parenthesis
                values = re.split(r"""\s+|,""", value)
                value = [value.strip() for value in values if value.strip()]
            term = dict(key=key,
                        seperator=seperator,
                        search_term=value)
        else:
            term = dict(key=None,
                        seperator=None,
                        search_term = item)

        items.append(term)
    print(pf(items))
    return(items)

if __name__ == '__main__':
    parse("foo=[3 2 1]")
    parse("foo=[3 2 1)")
    parse("foo=(3 2 1)")
    parse("shh")
    parse("shh grep")
    parse("LRS=(9 99 Chr4 122 155) cisLRS=(9 999 10)")
    parse("sal1 LRS=(9 99 Chr4 122 155) sal2 cisLRS=(9 999 10)")
    parse("sal1 sal3 LRS=(9 99 Chr4 122 155) wiki=bar sal2 go:foobar cisLRS=(9 999 10)")
    parse("sal1 LRS=(9 99 Chr4 122 155) wiki=bar sal2 go:foobar cisLRS=(9, 999, 10)")<commit_msg>Changed parser.py to deal with the wild card character (*) and to split by ">" and "<" in addition to "=" and ":"<commit_after>"""
Parses search terms input by user

Searches take two primary forms:
- search term by itself (ex. "shh" or "brain")
- key / separator / value(s) (ex. "LRS=(9 99 Chr4 122 155)" or "GO:342533")

In the example of "LRS=(9 99 Chr4 122 155)", the key is "LRS", the separator is "=" and the value
is everything within the parentheses.

Both "=" and ":" can be used as separators; in the future, it would also be good to allow no
separator at all (ex. "cisLRS(9 999 10)")

Both square brackets and parentheses can be used interchangeably. Both can also be used to
encapsulate a single value; "cisLRS=[9 999 10)" would
be acceptable.]

NEED TO DEAL WITH WILDCARD CHARACTER '*'

"""

from __future__ import print_function, division

import re

from pprint import pformat as pf

def parse(pstring):
    pstring = re.split(r"""(?:(\w+\s*=\s*[\(\[][^)]*[\)\]])  |  # LRS=(1 2 3), cisLRS=[4 5 6], etc
                       (\w+\s*[=:\>\<][\w\*]+)  |  # wiki=bar, GO:foobar, etc
                       ([\w\*]+))  # shh, brain, etc """, pstring,
                                                    flags=re.VERBOSE)
    pstring = [item.strip() for item in pstring if item and item.strip()]
    print(pstring)

    items = []

    separators = [re.escape(x) for x in (":", "=", "<", ">")]
    separators = '([%s])' % ("".join(separators))
    
    print("separators:", separators)

    for item in pstring:
        splat = re.split(separators, item)
        print("splat is:", splat)

        # splat is an array of 1 if no match, otherwise more than 1
        if len(splat) > 1:
            key, separator, value = splat
            if '(' in value or '[' in value:
                assert value.startswith(("(", "[")), "Invalid token"
                assert value.endswith((")", "]")), "Invalid token"
                value = value[1:-1] # Get rid of the parenthesis
                values = re.split(r"""\s+|,""", value)
                value = [value.strip() for value in values if value.strip()]
            term = dict(key=key,
                        separator=separator,
                        search_term=value)
        else:
            term = dict(key=None,
                        separator=None,
                        search_term = item)

        items.append(term)
    print(pf(items) + "\n")
    return(items)

if __name__ == '__main__':
    parse("foo=[3 2 1]")
    parse("WIKI=ho*")
    parse("LRS>9")
    parse("foo=[3 2 1)")
    parse("foo=(3 2 1)")
    parse("shh")
    parse("shh grep")
    parse("LRS=(9 99 Chr4 122 155) cisLRS=(9 999 10)")
    parse("sal1 LRS=(9 99 Chr4 122 155) sal2 cisLRS=(9 999 10)")
    parse("sal1 sal3 LRS=(9 99 Chr4 122 155) wiki=bar sal2 go:foobar cisLRS=(9 999 10)")
    parse("sal1 LRS=(9 99 Chr4 122 155) wiki=bar sal2 go:foobar cisLRS=(9, 999, 10)")
{'fraction_non_alphanumeric': 0.14933188876850848, 'fraction_numerical': 0.04622607439508848, 'mean_word_length': 2.2184776292852995, 'pattern_counts': {'":': 5, '<': 6, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4707688', 'n_tokens_mistral': 2072, 'n_tokens_neox': 1776, 'n_words': 603}