<commit_before># file is called AST to not collide with std lib module 'ast'
#
# It provides types to build ASTs in a simple lambda-notation style
#

import copy
import logging; logger = logging.getLogger(__name__)
import time

from StringIO import StringIO
from collections import deque

import networkx as nx

# TODO: move things depending on numpy (among others too) to a library file
import numpy as np
np_versions = map(int, np.__version__.split('.')[:2])


DEFAULT_MAX_PROGRAM_LEN = 100000


class PyllImportError(ImportError):
    """A pyll symbol was not defined in the scope """


class MissingArgument(object):
    """Object to represent a missing argument to a function application
    """


class SymbolTable(object):
    """
    An object whose methods generally allocate Apply nodes.

    _impls is a dictionary containing implementations for those nodes.

    >>> self.add(a, b)          # -- creates a new 'add' Apply node
    >>> self._impl['add'](a, b) # -- this computes a + b
    """

    def __init__(self):
        # -- list and dict are special because they are Python builtins
        self._impls = {
                'list': list,
                'dict': dict,
                'range': range,
                'len': len,
                'int': int,
                'float': float,
                'map': map,
                'max': max,
                'min': min,
                'getattr': getattr,
                }

    def _new_apply(self, name, args, kwargs, o_len, pure):
        pos_args = [as_apply(a) for a in args]
        named_args = [(k, as_apply(v)) for (k, v) in kwargs.items()]
        named_args.sort()
        return Apply(name,
                pos_args=pos_args,
                named_args=named_args,
                o_len=o_len,
                pure=pure)

    # ----

    def dict(self, *args, **kwargs):
        # XXX: figure out len
        return self._new_apply('dict', args, kwargs, o_len=None,
                pure=True)

    def int(self, arg):
        return self._new_apply('int', [as_apply(arg)], {}, o_len=None,
                pure=True)

    def float(self, arg):
        return self._new_apply('float', [as_apply(arg)], {}, o_len=None,
                pure=True)

    def len(self, obj):
        return self._new_apply('len', [obj], {}, o_len=None,
                pure=True)

    def list(self, init):
        return self._new_apply('list', [as_apply(init)], {}, o_len=None,
                pure=True)

    def map(self, fn, seq, pure=False):
        """
        pure - True is assertion that fn does not modify seq[i]
        """
        return self._new_apply('map', [as_apply(fn), as_apply(seq)], {},
                               o_len=seq.o_len,
                               pure=pure
                               )

    def range(self, *args):
        return self._new_apply('range', args, {}, o_len=None, pure=True)

    def max(self, *args):
        """ return max of args """
        return self._new_apply('max', map(as_apply, args), {},
                o_len=None, pure=True)

    def min(self, *args):
        """ return min of args """
        return self._new_apply('min', map(as_apply, args), {},
                o_len=None, pure=True)

    def getattr(self, obj, attr, *args):
        return self._new_apply('getattr',
                [as_apply(obj), as_apply(attr)] + map(as_apply, args),
                {},
                o_len=None,
                pure=True)

    # ----


    def _define(self, f, o_len, pure):
        name = f.__name__
        entry = SymbolTableEntry(self, name, o_len, pure)
        setattr(self, name, entry)
        self._impls[name] = f
        return f

    def define(self, f, o_len=None, pure=False):
        """Decorator for adding python functions to self
        """
        name = f.__name__
        if hasattr(self, name):
            raise ValueError('Cannot override existing symbol', name)
        return self._define(f, o_len, pure)

    def define_if_new(self, f, o_len=None, pure=False):
        """Pass silently if f matches the current implementation
        for f.__name__"""
        name = f.__name__
        if hasattr(self, name) and self._impls[name] is not f:
            raise ValueError('Cannot redefine existing symbol', name)
        return self._define(f, o_len, pure)

    def undefine(self, f):
        if isinstance(f, basestring):
            name = f
        else:
            name = f.__name__
        del self._impls[name]
        delattr(self, name)

    def define_pure(self, f):
        return self.define(f, o_len=None, pure=True)

    def define_info(self, o_len=None, pure=False):
        def wrapper(f):
            return self.define(f, o_len=o_len, pure=pure)
        return wrapper

    def inject(self, *args, **kwargs):
        """
        Add symbols from self into a dictionary and return the dict.

        This is used for import-like syntax: see `import_`.
        """
        rval = {}
        for k in args:
            try:
                rval[k] = getattr(self, k)
            except AttributeError:
                raise PyllImportError(k)
        for k, origk in kwargs.items():
            try:
                rval[k] = getattr(self, origk)
            except AttributeError:
                raise PyllImportError(origk)
        return rval

    def import_(self, _globals, *args, **kwargs):
        _globals.update(self.inject(*args, **kwargs))

class SymbolTableEntry(object):
    """A functools.partial-like class for adding symbol table entries.
    """
    def __init__(self, symbol_table, apply_name, o_len, pure):
        self.symbol_table = symbol_table
        self.apply_name = apply_name
        self.o_len = o_len
        self.pure = pure

    def __call__(self, *args, **kwargs):
        return self.symbol_table._new_apply(
                self.apply_name,
                args,
                kwargs,
                self.o_len,
                self.pure)

scope = SymbolTable()


def as_apply(obj):
    """Smart way of turning object into an Apply
    """
    if isinstance(obj, Apply):
        rval = obj
    elif isinstance(obj, tuple):
        rval = Apply('pos_args', [as_apply(a) for a in obj], {}, len(obj))
    elif isinstance(obj, list):
        rval = Apply('pos_args', [as_apply(a) for a in obj], {}, None)
    elif isinstance(obj, dict):
        items = obj.items()
        # -- should be fine to allow numbers and simple things
        #    but think about if it's ok to allow Applys
        #    it messes up sorting at the very least.
        items.sort()
        if all(isinstance(k, basestring) for k in obj):
            named_args = [(k, as_apply(v)) for (k, v) in items]
            rval = Apply('dict', [], named_args, len(named_args))
        else:
            new_items = [(k, as_apply(v)) for (k, v) in items]
            rval = Apply('dict', [as_apply(new_items)], {}, o_len=None)
    else:
        rval = Literal(obj)
    assert isinstance(rval, Apply)
    return rval


class Apply(object):
    """
    Represent a symbolic application of a symbol to arguments.

    o_len - None or int if the function is guaranteed to return a fixed number
        `o_len` of outputs if it returns successfully
    pure - True only if the function has no relevant side-effects
    """

    def __init__(self, name, pos_args, named_args,
            o_len=None,
            pure=False):
        self.name = name
        # -- tuples or arrays -> lists
        self.pos_args = list(pos_args)
        self.named_args = [[kw, arg] for (kw, arg) in named_args]
        # -- o_len is attached this early to support tuple unpacking and
        #    list coersion.
        self.o_len = o_len
        self.pure = pure
        assert all(isinstance(v, Apply) for v in pos_args)
        assert all(isinstance(v, Apply) for k, v in named_args)
        assert all(isinstance(k, basestring) for k, v in named_args)

    def eval(self, memo=None):
        """
        Recursively evaluate an expression graph.<commit_msg>Let Apply nodes cope with scope inconsistency.<commit_after># file is called AST to not collide with std lib module 'ast'
#
# It provides types to build ASTs in a simple lambda-notation style
#

import copy
import logging; logger = logging.getLogger(__name__)
import time

from StringIO import StringIO
from collections import deque

import networkx as nx

# TODO: move things depending on numpy (among others too) to a library file
import numpy as np
np_versions = map(int, np.__version__.split('.')[:2])


DEFAULT_MAX_PROGRAM_LEN = 100000


class PyllImportError(ImportError):
    """A pyll symbol was not defined in the scope """


class MissingArgument(object):
    """Object to represent a missing argument to a function application
    """


class SymbolTable(object):
    """
    An object whose methods generally allocate Apply nodes.

    _impls is a dictionary containing implementations for those nodes.

    >>> self.add(a, b)          # -- creates a new 'add' Apply node
    >>> self._impl['add'](a, b) # -- this computes a + b
    """

    def __init__(self):
        # -- list and dict are special because they are Python builtins
        self._impls = {
                'list': list,
                'dict': dict,
                'range': range,
                'len': len,
                'int': int,
                'float': float,
                'map': map,
                'max': max,
                'min': min,
                'getattr': getattr,
                }

    def _new_apply(self, name, args, kwargs, o_len, pure):
        pos_args = [as_apply(a) for a in args]
        named_args = [(k, as_apply(v)) for (k, v) in kwargs.items()]
        named_args.sort()
        return Apply(name,
                pos_args=pos_args,
                named_args=named_args,
                o_len=o_len,
                pure=pure)

    # ----

    def dict(self, *args, **kwargs):
        # XXX: figure out len
        return self._new_apply('dict', args, kwargs, o_len=None,
                pure=True)

    def int(self, arg):
        return self._new_apply('int', [as_apply(arg)], {}, o_len=None,
                pure=True)

    def float(self, arg):
        return self._new_apply('float', [as_apply(arg)], {}, o_len=None,
                pure=True)

    def len(self, obj):
        return self._new_apply('len', [obj], {}, o_len=None,
                pure=True)

    def list(self, init):
        return self._new_apply('list', [as_apply(init)], {}, o_len=None,
                pure=True)

    def map(self, fn, seq, pure=False):
        """
        pure - True is assertion that fn does not modify seq[i]
        """
        return self._new_apply('map', [as_apply(fn), as_apply(seq)], {},
                               o_len=seq.o_len,
                               pure=pure
                               )

    def range(self, *args):
        return self._new_apply('range', args, {}, o_len=None, pure=True)

    def max(self, *args):
        """ return max of args """
        return self._new_apply('max', map(as_apply, args), {},
                o_len=None, pure=True)

    def min(self, *args):
        """ return min of args """
        return self._new_apply('min', map(as_apply, args), {},
                o_len=None, pure=True)

    def getattr(self, obj, attr, *args):
        return self._new_apply('getattr',
                [as_apply(obj), as_apply(attr)] + map(as_apply, args),
                {},
                o_len=None,
                pure=True)

    # ----


    def _define(self, f, o_len, pure):
        name = f.__name__
        entry = SymbolTableEntry(self, name, o_len, pure)
        setattr(self, name, entry)
        self._impls[name] = f
        return f

    def define(self, f, o_len=None, pure=False):
        """Decorator for adding python functions to self
        """
        name = f.__name__
        if hasattr(self, name):
            raise ValueError('Cannot override existing symbol', name)
        return self._define(f, o_len, pure)

    def define_if_new(self, f, o_len=None, pure=False):
        """Pass silently if f matches the current implementation
        for f.__name__"""
        name = f.__name__
        if hasattr(self, name) and self._impls[name] is not f:
            raise ValueError('Cannot redefine existing symbol', name)
        return self._define(f, o_len, pure)

    def undefine(self, f):
        if isinstance(f, basestring):
            name = f
        else:
            name = f.__name__
        del self._impls[name]
        delattr(self, name)

    def define_pure(self, f):
        return self.define(f, o_len=None, pure=True)

    def define_info(self, o_len=None, pure=False):
        def wrapper(f):
            return self.define(f, o_len=o_len, pure=pure)
        return wrapper

    def inject(self, *args, **kwargs):
        """
        Add symbols from self into a dictionary and return the dict.

        This is used for import-like syntax: see `import_`.
        """
        rval = {}
        for k in args:
            try:
                rval[k] = getattr(self, k)
            except AttributeError:
                raise PyllImportError(k)
        for k, origk in kwargs.items():
            try:
                rval[k] = getattr(self, origk)
            except AttributeError:
                raise PyllImportError(origk)
        return rval

    def import_(self, _globals, *args, **kwargs):
        _globals.update(self.inject(*args, **kwargs))

class SymbolTableEntry(object):
    """A functools.partial-like class for adding symbol table entries.
    """
    def __init__(self, symbol_table, apply_name, o_len, pure):
        self.symbol_table = symbol_table
        self.apply_name = apply_name
        self.o_len = o_len
        self.pure = pure

    def __call__(self, *args, **kwargs):
        return self.symbol_table._new_apply(
                self.apply_name,
                args,
                kwargs,
                self.o_len,
                self.pure)

scope = SymbolTable()


def as_apply(obj):
    """Smart way of turning object into an Apply
    """
    if isinstance(obj, Apply):
        rval = obj
    elif isinstance(obj, tuple):
        rval = Apply('pos_args', [as_apply(a) for a in obj], {}, len(obj))
    elif isinstance(obj, list):
        rval = Apply('pos_args', [as_apply(a) for a in obj], {}, None)
    elif isinstance(obj, dict):
        items = obj.items()
        # -- should be fine to allow numbers and simple things
        #    but think about if it's ok to allow Applys
        #    it messes up sorting at the very least.
        items.sort()
        if all(isinstance(k, basestring) for k in obj):
            named_args = [(k, as_apply(v)) for (k, v) in items]
            rval = Apply('dict', [], named_args, len(named_args))
        else:
            new_items = [(k, as_apply(v)) for (k, v) in items]
            rval = Apply('dict', [as_apply(new_items)], {}, o_len=None)
    else:
        rval = Literal(obj)
    assert isinstance(rval, Apply)
    return rval


class Apply(object):
    """
    Represent a symbolic application of a symbol to arguments.

    o_len - None or int if the function is guaranteed to return a fixed number
        `o_len` of outputs if it returns successfully
    pure - True only if the function has no relevant side-effects
    """

    def __init__(self, name, pos_args, named_args,
            o_len=None,
            pure=False,
            define_params=None):
        self.name = name
        # -- tuples or arrays -> lists
        self.pos_args = list(pos_args)
        self.named_args = [[kw, arg] for (kw, arg) in named_args]
        # -- o_len is attached this early to support tuple unpacking and
        #    list coersion.
        self.o_len = o_len
        self.pure = pure
        # -- define_params lets us cope with stuff that may be in the
        #    SymbolTable on the master but not on the worker.
        self.define_params = define_params
        assert all(isinstance(v, Apply) for v in pos_args)
        assert all(isinstance(v, Apply) for k, v in named_args)
        assert all(isinstance(k, basestring) for k, v in named_args)

    def __setstate__(self, state):
        self.__dict__.update(state)
        # -- On deserialization, update scope if need be.
        if self.define_params:
            scope.define_if_new(**self.define_params)

    def eval(self, memo=None):
        """
        Recursively evaluate an expression graph.
{'fraction_non_alphanumeric': 0.1107062610068622, 'fraction_numerical': 0.0008501852189226938, 'mean_word_length': 1.987121349537457, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1032326', 'n_tokens_mistral': 4998, 'n_tokens_neox': 4791, 'n_words': 1498}