<commit_before>"""
Snuggs are s-expressions for Numpy.
"""

import functools
import itertools
import operator
import re
import sys

from pyparsing import (
    alphanums, ZeroOrMore, nums, oneOf, Word, Literal, Combine, QuotedString,
    ParseException, Forward, Group, CaselessLiteral, Optional, alphas,
    OneOrMore, ParseResults)

import numpy


__all__ = ['eval']
__version__ = "1.1.0"

# Python 2-3 compatibility
string_types = (str,) if sys.version_info[0] >= 3 else (basestring,)


class Context(object):

    def __init__(self):
        self._data = {}

    def add(self, name, val):
        self._data[name] = val

    def get(self, name):
        return self._data[name]

    def lookup(self, index, subindex=None):
        s = list(self._data.values())[int(index)-1]
        if subindex:
            return s[int(subindex)-1]
        else:
            return s

    def clear(self):
        self._data = {}

_ctx = Context()


class ctx(object):

    def __init__(self, **kwds):
        self.kwds = kwds

    def __enter__(self):
        _ctx.clear()
        for k, v in self.kwds.items():
            _ctx.add(k, v)
        return self

    def __exit__(self, exc_type=None, exc_val=None, exc_tb=None):
        self.kwds = None
        _ctx.clear()


op_map = {
    '*': operator.mul,
    '+': operator.add,
    '/': operator.truediv if sys.version_info[0] >= 3 else operator.div,
    '-': operator.sub,
    '<': operator.lt,
    '<=': operator.le,
    '==': operator.eq,
    '!=': operator.ne,
    '>=': operator.ge,
    '>': operator.gt,
    '&': operator.and_,
    '|': operator.or_
    }

def asarray(*args):
    if len(args) == 1 and hasattr(args[0], '__iter__'):
        return numpy.asanyarray(list(args[0]))
    else:
        return numpy.asanyarray(list(args))

func_map = {
    'asarray': asarray,
    'read': _ctx.lookup,
    'take': lambda a, idx: numpy.take(a, idx-1, axis=0),
    }

higher_func_map = {
    'map': itertools.map if sys.version_info[0] >= 3 else itertools.imap,
    'partial': functools.partial,
    }

# Definition of the grammar.
decimal = Literal('.')
e = CaselessLiteral('E')
sign = Literal('+') | Literal('-')
number = Word(nums)
name = Word(alphas)

var = name.setParseAction(
    lambda s, l, t: _ctx.get(t[0]))

integer = Combine(
    Optional(sign) +
    number
    ).setParseAction(lambda s, l, t: int(t[0]))
<commit_msg>map on python 3.<commit_after>"""
Snuggs are s-expressions for Numpy.
"""

import functools
import itertools
import operator
import re
import sys

from pyparsing import (
    alphanums, ZeroOrMore, nums, oneOf, Word, Literal, Combine, QuotedString,
    ParseException, Forward, Group, CaselessLiteral, Optional, alphas,
    OneOrMore, ParseResults)

import numpy


__all__ = ['eval']
__version__ = "1.1.0"

# Python 2-3 compatibility
string_types = (str,) if sys.version_info[0] >= 3 else (basestring,)


class Context(object):

    def __init__(self):
        self._data = {}

    def add(self, name, val):
        self._data[name] = val

    def get(self, name):
        return self._data[name]

    def lookup(self, index, subindex=None):
        s = list(self._data.values())[int(index)-1]
        if subindex:
            return s[int(subindex)-1]
        else:
            return s

    def clear(self):
        self._data = {}

_ctx = Context()


class ctx(object):

    def __init__(self, **kwds):
        self.kwds = kwds

    def __enter__(self):
        _ctx.clear()
        for k, v in self.kwds.items():
            _ctx.add(k, v)
        return self

    def __exit__(self, exc_type=None, exc_val=None, exc_tb=None):
        self.kwds = None
        _ctx.clear()


op_map = {
    '*': operator.mul,
    '+': operator.add,
    '/': operator.truediv if sys.version_info[0] >= 3 else operator.div,
    '-': operator.sub,
    '<': operator.lt,
    '<=': operator.le,
    '==': operator.eq,
    '!=': operator.ne,
    '>=': operator.ge,
    '>': operator.gt,
    '&': operator.and_,
    '|': operator.or_
    }

def asarray(*args):
    if len(args) == 1 and hasattr(args[0], '__iter__'):
        return numpy.asanyarray(list(args[0]))
    else:
        return numpy.asanyarray(list(args))

func_map = {
    'asarray': asarray,
    'read': _ctx.lookup,
    'take': lambda a, idx: numpy.take(a, idx-1, axis=0),
    }

higher_func_map = {
    'map': map if sys.version_info[0] >= 3 else itertools.imap,
    'partial': functools.partial,
    }

# Definition of the grammar.
decimal = Literal('.')
e = CaselessLiteral('E')
sign = Literal('+') | Literal('-')
number = Word(nums)
name = Word(alphas)

var = name.setParseAction(
    lambda s, l, t: _ctx.get(t[0]))

integer = Combine(
    Optional(sign) +
    number
    ).setParseAction(lambda s, l, t: int(t[0]))

{'fraction_non_alphanumeric': 0.15615108672715763, 'fraction_numerical': 0.008651614264612787, 'mean_word_length': 2.75594294770206, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2440304', 'n_tokens_mistral': 1722, 'n_tokens_neox': 1679, 'n_words': 395}