<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""My own special query language for iris, because I'm an idiot.

See also, some interesting notes on implementing various types of simple
command grammars using gnu/readline & python:
    https://www.ironalbatross.net/wiki/index.php5?title=Python_Readline_Completions
    https://www.ironalbatross.net/wiki/index.php5?title=Python_Cmd_Completions

Something even more complex might eventually be needed.

If you can't read the LEPL code below, this is a rough BNF translation:

<alpha> ::= { a..z | A..Z }
<num> ::= { 0..9 }
<alphanum> ::= { alpha | num }
<initialalpha> ::= alpha { alphanum }

# tokens
find ::= find
count ::= count
where ::= where
tag   ::= tag
and   ::= (AND | &)
or    ::= (OR | "|")
field ::= <initialalpha>
EOL ::= "\n"

# data
string ::= '"' { <alphanum> } '"'
number ::= { <num> } [ "." { <num> }]
list   ::= "(" ( string | number ) { "," ( string | number ) } ")"
literal ::= ( string | number | list )

# comparisons
equal ::= "="           # implied '*foo*' for strings
doubleequal ::= "=="    # precisely equal
lt ::= "<"
gt ::= ">"
lte ::= lt equal
gte ::= gt equal
comps ::= ( equal | doubleequal | lt | gt | gte | lte )

in  ::= "in"

# expressions
field_list ::= "(" field { "," field} ")"
comparison ::= field comps literal
in_expr ::= field in list
where_clause ::= where ( comparison | in_expr ) [ { (and | or) (comparison | in_expr ) } ]

# statements
find_stmt ::= find [number] [field_list] [where_clause] EOL
count_stmt ::= count [where_clause] EOL
tag_stmt ::= tag string where_clause EOL

"""

import readline
import cmd
import sys
import re

from iris import version
from iris.utils import color, bold, white, green, red

from lepl import *

import logging
logger = logging.getLogger("lepl")
logger.setLevel(logging.CRITICAL)

class Token(object):
    tokens = ('find', 'count', 'tag', 'where', 'and', 'or', 'in')
    def __init__(self, type, value):
        self.type = type.lower()
        self.value = value[0] if isinstance(value, list) else value

    def __repr__(self):
        if self.type in self.tokens:
            return '<%s>' % self.type
        if self.type == 'field':
            return '<:%s>' % self.value
        return '<%s (%r)>' % (self.type, self.value)
    def __str__(self): return str(self.value)

def token(name):
    """Generates a function that returns a curried token."""
    return lambda value: Token(name, value)

def Insensitive(string):
    """A case insensitive literal."""
    return Regexp(re.compile(string, re.I))

def fold(func):
    return lambda x: func(x[0])

def numerify(x):
    value = x[0]
    try:
        return int(value)
    except ValueError:
        return float(value)

# types
Alpha = lambda: Word(Upper() | Lower())
AlphaNum = lambda: Word(Alpha() | Digit())
InitialAlpha = lambda: Word(Alpha(), (AlphaNum() | Any("-_")))

# tokens
find    = Insensitive("find")               > token('find')
count   = Insensitive("count")              > token('count')
tag     = Insensitive("tag")                > token('tag')
where   = Insensitive("where")              > token('where')
AND     = Insensitive("and") | Literal("&")     > token('and')
OR      = Insensitive("or")  | Literal("|")     > token('or')
field   = InitialAlpha()    > token('field')
eol     = Eos()

# data types
string  = SingleLineString() | SingleLineString(quote="'")
number  = Float()           > numerify
with DroppedSpace():
    list_   = Drop("(") & (string | number)[0:, Drop(Whitespace() | ",")[1:]] & Drop(")") > list

# operators
in_oper = Insensitive("in")     > token('in')
equal   = Literal("=")          > token('equal')
dblequal = Literal("==")        > token('dblequal')
lt      = Literal("<")          > token('lt')
lte     = Literal("<=")         > token('lte')
gt      = Literal(">")          > token('gt')
gte     = Literal(">=")         > token('gte')

number_operator = dblequal | equal | lte | gte | lt | gt
string_operator = in_oper | equal | dblequal
list_operator   = in_oper

# expressions
with DroppedSpace():
    field_list  = Drop("(") & field[1:, Drop(Whitespace() | ",")[1:]] & Drop(")") > list
    num_expr    = number_operator & number
    string_expr = string_operator & string
    list_expr   = list_operator & list_
    logic_expr  = field & (num_expr | string_expr | list_expr)
    combin_expr = Drop(Whitespace())[:] & (AND | OR) & Drop(Whitespace())[:]
    where_clause = logic_expr[1:, combin_expr]
    where_expr  = where & where_clause
    where_expr.config.auto_memoize()

# statements<commit_msg>simplify the separators with a new Separator composed type, add comma literal, make empty lists illegal<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""My own special query language for iris, because I'm an idiot.

See also, some interesting notes on implementing various types of simple
command grammars using gnu/readline & python:
    https://www.ironalbatross.net/wiki/index.php5?title=Python_Readline_Completions
    https://www.ironalbatross.net/wiki/index.php5?title=Python_Cmd_Completions

Something even more complex might eventually be needed.

If you can't read the LEPL code below, this is a rough BNF translation:

<alpha> ::= { a..z | A..Z }
<num> ::= { 0..9 }
<alphanum> ::= { alpha | num }
<initialalpha> ::= alpha { alphanum }

# tokens
find ::= find
count ::= count
where ::= where
tag   ::= tag
and   ::= (AND | &)
or    ::= (OR | "|")
field ::= <initialalpha>
EOL ::= "\n"

# data
string ::= '"' { <alphanum> } '"'
number ::= { <num> } [ "." { <num> }]
list   ::= "(" ( string | number ) { "," ( string | number ) } ")"
literal ::= ( string | number | list )

# comparisons
equal ::= "="           # implied '*foo*' for strings
doubleequal ::= "=="    # precisely equal
lt ::= "<"
gt ::= ">"
lte ::= lt equal
gte ::= gt equal
comps ::= ( equal | doubleequal | lt | gt | gte | lte )

in  ::= "in"

# expressions
field_list ::= "(" field { "," field} ")"
comparison ::= field comps literal
in_expr ::= field in list
where_clause ::= where ( comparison | in_expr ) [ { (and | or) (comparison | in_expr ) } ]

# statements
find_stmt ::= find [number] [field_list] [where_clause] EOL
count_stmt ::= count [where_clause] EOL
tag_stmt ::= tag string where_clause EOL

"""

import readline
import cmd
import sys
import re

from iris import version
from iris.utils import color, bold, white, green, red

from lepl import *

import logging
logger = logging.getLogger("lepl")
logger.setLevel(logging.CRITICAL)

class Token(object):
    tokens = ('find', 'count', 'tag', 'where', 'and', 'or', 'in')
    def __init__(self, type, value):
        self.type = type.lower()
        self.value = value[0] if isinstance(value, list) else value

    def __repr__(self):
        if self.type in self.tokens:
            return '<%s>' % self.type
        if self.type == 'field':
            return '<:%s>' % self.value
        return '<%s (%r)>' % (self.type, self.value)
    def __str__(self): return str(self.value)

def token(name):
    """Generates a function that returns a curried token."""
    return lambda value: Token(name, value)

def Insensitive(string):
    """A case insensitive literal."""
    return Regexp(re.compile(string, re.I))

def Separator(matcher, drop=True):
    if drop:
        return Drop(Whitespace()[:]) & Drop(matcher) & Drop(Whitespace()[:])
    return Drop(Whitespace())[:] & matcher & Drop(Whitespace())[:]

def fold(func):
    return lambda x: func(x[0])

def numerify(x):
    value = x[0]
    try:
        return int(value)
    except ValueError:
        return float(value)

# types
Alpha = lambda: Word(Upper() | Lower())
AlphaNum = lambda: Word(Alpha() | Digit())
InitialAlpha = lambda: Word(Alpha(), (AlphaNum() | Any("-_")))

# tokens
find    = Insensitive("find")               > token('find')
count   = Insensitive("count")              > token('count')
tag     = Insensitive("tag")                > token('tag')
where   = Insensitive("where")              > token('where')
AND     = Insensitive("and") | Literal("&")     > token('and')
OR      = Insensitive("or")  | Literal("|")     > token('or')
field   = InitialAlpha()    > token('field')
comma   = Literal(",")
eol     = Eos()

# data types
string  = String()
number  = Float()           > numerify
with DroppedSpace():
    list_   = Drop("(") & (string | number)[1:, Separator(comma)] & Drop(")") > list

# operators
in_oper = Insensitive("in")     > token('in')
equal   = Literal("=")          > token('equal')
dblequal = Literal("==")        > token('dblequal')
lt      = Literal("<")          > token('lt')
lte     = Literal("<=")         > token('lte')
gt      = Literal(">")          > token('gt')
gte     = Literal(">=")         > token('gte')

number_operator = dblequal | equal | lte | gte | lt | gt
string_operator = in_oper | equal | dblequal
list_operator   = in_oper

# expressions
with DroppedSpace():
    field_list  = Drop("(") & field[1:, Separator(comma)] & Drop(")") > list
    num_expr    = number_operator & number
    string_expr = string_operator & string
    list_expr   = list_operator & list_
    logic_expr  = field & (num_expr | string_expr | list_expr)
    where_clause = logic_expr[1:, Separator(AND | OR, drop=False)]
    where_expr  = where & where_clause
    where_expr.config.auto_memoize()

# statements
{'fraction_non_alphanumeric': 0.17018993454233286, 'fraction_numerical': 0.002575383624852452, 'mean_word_length': 3.234438891412994, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 65, 'https://': 4, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4388098', 'n_tokens_mistral': 3114, 'n_tokens_neox': 3087, 'n_words': 835}