<commit_before>from __future__ import unicode_literals, print_function
import re, itertools, operator, ast
from pypeg2 import *

# This is needed because using a plain 'not' will remove it from the ast when
# parsed.
RE_NOT = re.compile('not')

# And this is needed because otherwise "not" will be usable as an Indentifier.
K('not')

def py_bool_to_lit(py_bool):
    return parse( 'true' if py_bool else 'false', BooleanLiteral)

class Identifier(str):
    grammar = word

class StringLiteral(str):

    def __new__(cls, s):
        return super(StringLiteral, cls).__new__(cls, ast.literal_eval(s))

    grammar = [re.compile(r'"[^\\\n\r]+?"'), re.compile(r"'[^\\\n\r]+?'")]

class IntegerLiteral(int):
    grammar = re.compile(r'-?\d+')

class FloatLiteral(float):
    grammar = re.compile(r'-?\d+.\d+')

class BooleanLiteral(Keyword):
    grammar = Enum(K('true'), K('false'))

class NullLiteral(Keyword):
    grammar = Enum(K('null'))

Comparable = [NullLiteral, FloatLiteral, IntegerLiteral, 
                                StringLiteral, Identifier]


class ComparisonOperator(str):
    grammar = re.compile(r'=|>|<|!=|>=|<=')

class BooleanFunctionName(Keyword):
    grammar = Enum(K('and'), K('or'))

class ComparisonOperation(List):
    pass

ComparisonOperation.grammar = (
        Comparable,
        blank,
        attr('comp_op', ComparisonOperator),
        blank,
        Comparable,
)

class BooleanOperationSimple(List):
# The flag() pypeg2 function works great when parsing but does not work when
# composing (the flag gets output whether it was in the source text or not. So
# a workaround is this:
    grammar = (
            attr('negated', optional(RE_NOT)),
            ComparisonOperation,
    )

class BooleanOperation(List):
    pass

BooleanOperation.grammar = (
        BooleanOperationSimple,
        maybe_some(
            blank,
            BooleanFunctionName,
            blank,
            BooleanOperationSimple,
        ),
)

class Expression(List):
    pass

Expression.grammar = (
        [BooleanOperationSimple, ('(', Expression, ')')],
        maybe_some(
            blank,
            BooleanFunctionName,
            blank,
            [BooleanOperationSimple, ('(', Expression, ')')],
        ),
)

class LangInstance:
	pass

def create_lang_instance(var_map = None):
	"""
	>>> lang_instance = create_lang_instance()
	>>> lang_instance.aml_evaluate(lang_instance.aml_compile('1 = 1'))
	True
	>>> li = create_lang_instance()
	>>> c = li.aml_compile
	>>> e = li.aml_evaluate
	>>> p = li.aml_translate_python
	>>> s = li.aml_translate_sql
	>>> u = li.aml_suggest
	>>> e(c('1 = 0'))
	False
	>>> e(c('"1" = "1"'))
	True
	>>> e(c('(1=1)'))
	True
	>>> e(c('1 > 1'))
	False
	>>> e(c('not 1 > 1'))
	True
	>>> e(c('1 != 1'))
	False
	>>> e(c('-2 = -2'))
	True
	>>> eval(p(c('-2 = -2')))
	True
	>>> eval(p(c('null = null')))
	True
	>>> eval(p(c('1 = null')))
	False
	>>> e(c('"foo" = "foo"'))
	True
	>>> e(c('"foo" = \\'foo\\''))
	True
	>>> e(c('"fo\\'o" = "fo\\'o"'))
	True
	>>> e(c("'foo'" + '=' + '"foo"'))
	True
	>>> li = create_lang_instance({'foo' : 1});
	>>> c = li.aml_compile
	>>> e = li.aml_evaluate
	>>> e(c('foo = 1'))
	True
	>>> li = create_lang_instance({'foo' : 1.00})
	>>> c = li.aml_compile
	>>> e = li.aml_evaluate
	>>> e(c('foo = 1'))
	True
	>>> li = create_lang_instance({'foo' : 2.24})
	>>> c = li.aml_compile
	>>> e = li.aml_evaluate
	>>> e(c('foo = 2.24'))
	True
	>>> li = create_lang_instance({'foo' : 'foo'})
	>>> c = li.aml_compile
	>>> e = li.aml_evaluate
	>>> e(c('foo = "foo"'))
	True
	>>> li = create_lang_instance()
	>>> c = li.aml_compile
	>>> p = li.aml_translate_python
	>>> s = li.aml_translate_sql
	>>> s(c('null = null'))
	u'null is null'
	>>> p(c('null = null'))
	u'None == None'
	>>> s(c('null != null'))
	u'null is not null'
	>>> p(c('null != null'))
	u'None != None'
	>>> s(c('5 != 3'))
	u'5 <> 3'
	>>> p(c('5 != 3'))
	u'5 != 3'
	>>> li = create_lang_instance({'foo' : 'bar', 'fo2' : 'ba2'})
	>>> u = li.aml_suggest
	>>> u('1 = fo')
	[u'fo2', u'foo']
	>>> u('1 = FO')
	[u'fo2', u'foo']
	"""

	def eval_node(node):
		en = lambda n: eval_node(n)
		if isinstance(node, Identifier):<commit_msg>SyntaxError will be given when invalid identifier.<commit_after>from __future__ import unicode_literals, print_function
import re, itertools, operator, ast
from pypeg2 import *

# This is needed because using a plain 'not' will remove it from the ast when
# parsed.
RE_NOT = re.compile('not')

# And this is needed because otherwise "not" will be usable as an Indentifier.
K('not')


class LangInstance:
	pass

def create_lang_instance(var_map = None):
	"""
	>>> lang_instance = create_lang_instance()
	>>> lang_instance.aml_evaluate(lang_instance.aml_compile('1 = 1'))
	True
	>>> li = create_lang_instance()
	>>> c = li.aml_compile
	>>> e = li.aml_evaluate
	>>> p = li.aml_translate_python
	>>> s = li.aml_translate_sql
	>>> u = li.aml_suggest
	>>> e(c('1 = 0'))
	False
	>>> e(c('"1" = "1"'))
	True
	>>> e(c('(1=1)'))
	True
	>>> e(c('1 > 1'))
	False
	>>> e(c('not 1 > 1'))
	True
	>>> e(c('1 != 1'))
	False
	>>> e(c('-2 = -2'))
	True
	>>> eval(p(c('-2 = -2')))
	True
	>>> eval(p(c('null = null')))
	True
	>>> eval(p(c('1 = null')))
	False
	>>> e(c('"foo" = "foo"'))
	True
	>>> e(c('"foo" = \\'foo\\''))
	True
	>>> e(c('"fo\\'o" = "fo\\'o"'))
	True
	>>> e(c("'foo'" + '=' + '"foo"'))
	True
	>>> li = create_lang_instance({'foo' : 1});
	>>> c = li.aml_compile
	>>> e = li.aml_evaluate
	>>> e(c('foo = 1'))
	True
	>>> li = create_lang_instance({'foo' : 1.00})
	>>> c = li.aml_compile
	>>> e = li.aml_evaluate
	>>> e(c('foo = 1'))
	True
	>>> li = create_lang_instance({'foo' : 2.24})
	>>> c = li.aml_compile
	>>> e = li.aml_evaluate
	>>> e(c('foo = 2.24'))
	True
	>>> li = create_lang_instance({'foo' : 'foo'})
	>>> c = li.aml_compile
	>>> e = li.aml_evaluate
	>>> e(c('foo = "foo"'))
	True
	>>> li = create_lang_instance()
	>>> c = li.aml_compile
	>>> p = li.aml_translate_python
	>>> s = li.aml_translate_sql
	>>> s(c('null = null'))
	u'null is null'
	>>> p(c('null = null'))
	u'None == None'
	>>> s(c('null != null'))
	u'null is not null'
	>>> p(c('null != null'))
	u'None != None'
	>>> s(c('5 != 3'))
	u'5 <> 3'
	>>> p(c('5 != 3'))
	u'5 != 3'
	>>> li = create_lang_instance({'foo' : 'bar', 'fo2' : 'ba2'})
	>>> c = li.aml_compile
	>>> p = li.aml_translate_python
	>>> e = li.aml_evaluate
	>>> u = li.aml_suggest
	>>> u('1 = fo')
	[u'fo2', u'foo']
	>>> u('1 = FO')
	[u'fo2', u'foo']
	>>> p(c('null = null'))
	u'None == None'
	>>> e(c('foo = "bar"'))
	True
	>>> e(c('fo2 = "ba2"'))
	True
	>>> c('abc = 1')
	Traceback (most recent call last):
	SyntaxError: expecting one of [<class '__init__.BooleanOperationSimple'>, (u'(', <class '__init__.Expression'>, u')')] (line 1)
	"""

	def py_bool_to_lit(py_bool):
		return parse( 'true' if py_bool else 'false', BooleanLiteral)

	if not var_map:
		class Identifier(str):
			grammar = re.compile(r'$a') # This will match nothing.
	else:
		class Identifier(Keyword):
			grammar = Enum(*[K(v) for v in var_map.iterkeys()])
		
	class StringLiteral(str):

		def __new__(cls, s):
			return super(StringLiteral, cls).__new__(cls, ast.literal_eval(s))

		grammar = [re.compile(r'"[^\\\n\r]+?"'), re.compile(r"'[^\\\n\r]+?'")]

	class IntegerLiteral(int):
		grammar = re.compile(r'-?\d+')

	class FloatLiteral(float):
		grammar = re.compile(r'-?\d+.\d+')

	class BooleanLiteral(Keyword):
		grammar = Enum(K('true'), K('false'))

	class NullLiteral(Keyword):
		grammar = Enum(K('null'))

	Comparable = [NullLiteral, FloatLiteral, IntegerLiteral, 
									StringLiteral, Identifier]


	class ComparisonOperator(str):
		grammar = re.compile(r'=|>|<|!=|>=|<=')

	class BooleanFunctionName(Keyword):
		grammar = Enum(K('and'), K('or'))

	class ComparisonOperation(List):
		pass

	ComparisonOperation.grammar = (
			Comparable,
			blank,
			attr('comp_op', ComparisonOperator),
			blank,
			Comparable,
	)

	class BooleanOperationSimple(List):
# The flag() pypeg2 function works great when parsing but does not work when
# composing (the flag gets output whether it was in the source text or not. So
# a workaround is this:
		grammar = (
				attr('negated', optional(RE_NOT)),
				ComparisonOperation,
		)

	class BooleanOperation(List):
		pass

	BooleanOperation.grammar = (
			BooleanOperationSimple,
			maybe_some(
				blank,
				BooleanFunctionName,
				blank,
				BooleanOperationSimple,
			),
	)

	class Expression(List):
		pass

	Expression.grammar = (
			[BooleanOperationSimple, ('(', Expression, ')')],
			maybe_some(
				blank,
				BooleanFunctionName,
				blank,
				[BooleanOperationSimple, ('(', Expression, ')')],
			),
	)

	def eval_node(node):
		en = lambda n: eval_node(n)
		if isinstance(node, Identifier):
{'fraction_non_alphanumeric': 0.20589584767148428, 'fraction_numerical': 0.011241110346409728, 'mean_word_length': 3.6353003721424773, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 348, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1939081', 'n_tokens_mistral': 3458, 'n_tokens_neox': 3260, 'n_words': 730}