<commit_before>import operator
import math
import waxeye
from parsers.parser import Parser
from itertools import chain
import functools


axes = {
    'ancestor'              : lambda context, field_filter: exception(field_filter),
    'ancestor-or-self'      : lambda context, field_filter: exception(field_filter),
    'attribute'             : lambda context, field_filter: axis_attribute(context, field_filter),
    'child'                 : lambda context, field_filter: axis_child(context, field_filter),
    'descendant'            : lambda context, field_filter: axis_descendant(context, field_filter),
    'descendant-or-self'    : lambda context, field_filter: axis_descendant_or_self(context, field_filter),
    'following'             : lambda context, field_filter: exception(field_filter),
    'following-sibling'     : lambda context, field_filter: exception(field_filter),
    'namespace'             : lambda context, field_filter: exception(field_filter),
    'parent'                : lambda context, field_filter: axis_parent(context, field_filter),
    'preceding'             : lambda context, field_filter: exception(field_filter),
    'preceding-sibling'     : lambda context, field_filter: exception(field_filter),
    'self'                  : lambda context, field_filter: axis_self(context, field_filter),
    '@'                     : lambda context, field_filter: axis_attribute(context, field_filter)
}


matchers = {
    'path'                  : lambda node, context: match_path(node, context),
    'node_test'             : lambda node, context: match_node_test(node, context),
    'predicate'             : lambda node, context: eval_predicate(node, context),
    'predicate_expr'        : lambda node, context: matchers[node.children[0].type](node.children[0], context),
    'or_expr'               : lambda node, context: match_or_expr(node, context),
    'and_expr'              : lambda node, context: match_and_expr(node, context),
    'equality_expr'         : lambda node, context: eval_bin_op(node, context),
    'relational_expr'       : lambda node, context: eval_bin_op(node, context),
    'additive_expr'         : lambda node, context: eval_bin_op(node, context),
    'multiplicative_expr'   : lambda node, context: eval_bin_op(node, context),
    'primary_expr'          : lambda node, context: match_primary_expr(node, context),
    'unary_expr'            : lambda node, context: match_unary_expr(node, context),
    'argument'              : lambda node, context: eval_unary_expr(node.children[0], context)
}


functions = {
    'last'                  : lambda context, *args: context['size'],
    'position'              : lambda context, *args: context['pos'],
    'count'                 : lambda context, *args: len(args[0]),
    'local-name'            : lambda context, *args: "[!!NOT IMPLEMENTED YET!!]",
    'name'                  : lambda context, *args: "[!!NOT IMPLEMENTED YET!!]",
    'string'                : lambda context, *args: unicode(args[0]),
    'concat'                : lambda context, *args: ''.join(args),
    'starts-with'           : lambda context, *args: "[!!NOT IMPLEMENTED YET!!]",
    'contains'              : lambda context, *args: args[1] in args[0],
    'substring-before'      : lambda context, *args: substring_before(context, *args),
    'substring-after'       : lambda context, *args: substring_after(context, *args),
    'substring'             : lambda context, *args: "[!!NOT IMPLEMENTED YET!!]",
    'string-length'         : lambda context, *args: len(args[0]),
    'normalize-space'       : lambda context, *args: "[!!NOT IMPLEMENTED YET!!]",
    'translate'             : lambda context, *args: "[!!NOT IMPLEMENTED YET!!]",
    'boolean'               : lambda context, *args: bool(args[0]),
    'not'                   : lambda context, *args: not bool(args[0]),
    'true'                  : lambda context, *args: True,
    'false'                 : lambda context, *args: False,
    'lang'                  : lambda context, *args: False,
    'number'                : lambda context, *args: "[!!NOT IMPLEMENTED YET!!]",
    'sum'                   : lambda context, *args: sum((float(x['value']) for x in args[0])),
    'floor'                 : lambda context, *args: math.floor(args[0]),
    'ceiling'               : lambda context, *args: math.ceil(args[0]),
    'round'                 : lambda context, *args: round(args[0])
}


operators = {
    '+'         : operator.add,
    '-'         : operator.sub,
    '*'         : operator.mul,
    'd'         : operator.div,
    'm'         : operator.mod,
    '='         : operator.eq,
    '!='        : operator.ne,
    '>'         : operator.gt,
    '<'         : operator.lt,
    '>='        : operator.ge,
    '<='        : operator.le
}


class InvalidQueryException(Exception):
    pass<commit_msg>added implementation for starts-with and number functions.<commit_after>import operator
import math
import waxeye
from parsers.parser import Parser
from itertools import chain
import functools


axes = {
    'ancestor'              : lambda context, field_filter: exception(field_filter),
    'ancestor-or-self'      : lambda context, field_filter: exception(field_filter),
    'attribute'             : lambda context, field_filter: axis_attribute(context, field_filter),
    'child'                 : lambda context, field_filter: axis_child(context, field_filter),
    'descendant'            : lambda context, field_filter: axis_descendant(context, field_filter),
    'descendant-or-self'    : lambda context, field_filter: axis_descendant_or_self(context, field_filter),
    'following'             : lambda context, field_filter: exception(field_filter),
    'following-sibling'     : lambda context, field_filter: exception(field_filter),
    'namespace'             : lambda context, field_filter: exception(field_filter),
    'parent'                : lambda context, field_filter: axis_parent(context, field_filter),
    'preceding'             : lambda context, field_filter: exception(field_filter),
    'preceding-sibling'     : lambda context, field_filter: exception(field_filter),
    'self'                  : lambda context, field_filter: axis_self(context, field_filter),
    '@'                     : lambda context, field_filter: axis_attribute(context, field_filter)
}


matchers = {
    'path'                  : lambda node, context: match_path(node, context),
    'node_test'             : lambda node, context: match_node_test(node, context),
    'predicate'             : lambda node, context: eval_predicate(node, context),
    'predicate_expr'        : lambda node, context: matchers[node.children[0].type](node.children[0], context),
    'or_expr'               : lambda node, context: match_or_expr(node, context),
    'and_expr'              : lambda node, context: match_and_expr(node, context),
    'equality_expr'         : lambda node, context: eval_bin_op(node, context),
    'relational_expr'       : lambda node, context: eval_bin_op(node, context),
    'additive_expr'         : lambda node, context: eval_bin_op(node, context),
    'multiplicative_expr'   : lambda node, context: eval_bin_op(node, context),
    'primary_expr'          : lambda node, context: match_primary_expr(node, context),
    'unary_expr'            : lambda node, context: match_unary_expr(node, context),
    'argument'              : lambda node, context: eval_unary_expr(node.children[0], context)
}


functions = {
    'last'                  : lambda context, *args: context['size'],
    'position'              : lambda context, *args: context['pos'],
    'count'                 : lambda context, *args: len(args[0]),
    'local-name'            : lambda context, *args: "[!!NOT IMPLEMENTED YET!!]",
    'name'                  : lambda context, *args: "[!!NOT IMPLEMENTED YET!!]",
    'string'                : lambda context, *args: unicode(args[0]),
    'concat'                : lambda context, *args: ''.join(args),
    'starts-with'           : lambda context, *args: args[0].startswith(args[1]),
    'contains'              : lambda context, *args: args[1] in args[0],
    'substring-before'      : lambda context, *args: substring_before(context, *args),
    'substring-after'       : lambda context, *args: substring_after(context, *args),
    'substring'             : lambda context, *args: "[!!NOT IMPLEMENTED YET!!]",
    'string-length'         : lambda context, *args: len(args[0]),
    'normalize-space'       : lambda context, *args: "[!!NOT IMPLEMENTED YET!!]",
    'translate'             : lambda context, *args: "[!!NOT IMPLEMENTED YET!!]",
    'boolean'               : lambda context, *args: bool(args[0]),
    'not'                   : lambda context, *args: not bool(args[0]),
    'true'                  : lambda context, *args: True,
    'false'                 : lambda context, *args: False,
    'lang'                  : lambda context, *args: False,
    'number'                : lambda context, *args: float(args[0]),
    'sum'                   : lambda context, *args: sum((float(x['value']) for x in args[0])),
    'floor'                 : lambda context, *args: math.floor(args[0]),
    'ceiling'               : lambda context, *args: math.ceil(args[0]),
    'round'                 : lambda context, *args: round(args[0])
}


operators = {
    '+'         : operator.add,
    '-'         : operator.sub,
    '*'         : operator.mul,
    'd'         : operator.div,
    'm'         : operator.mod,
    '='         : operator.eq,
    '!='        : operator.ne,
    '>'         : operator.gt,
    '<'         : operator.lt,
    '>='        : operator.ge,
    '<='        : operator.le
}


class InvalidQueryException(Exception):
    pass
{'fraction_non_alphanumeric': 0.1318107333060221, 'fraction_numerical': 0.003174928308070463, 'mean_word_length': 2.4071877180739705, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1928254', 'n_tokens_mistral': 2823, 'n_tokens_neox': 2802, 'n_words': 677}