<commit_before>from __future__ import absolute_import, division, print_function

import toolz
import datashape
import functools
from toolz import concat, memoize, partial, pipe
from toolz.curried import map, filter
import re

from datashape import dshape, DataShape, Record, Var, Mono
from datashape.predicates import isscalar, iscollection, isboolean, isrecord

from ..compatibility import _strtypes, builtins
from .core import *
from .method_dispatch import select_functions
from ..dispatch import dispatch

__all__ = ['Expr', 'ElemWise', 'Field', 'Symbol', 'discover', 'Projection',
           'projection', 'Selection', 'selection', 'Label', 'label', 'Map',
           'ReLabel', 'relabel', 'Apply', 'Slice', 'shape', 'ndim', 'label',
           'symbol']


def isvalid_identifier(s):
    """

    >>> isvalid_identifier('Hello')
    True
    >>> isvalid_identifier('Hello world')
    False
    >>> isvalid_identifier('Helloworld!')
    False
    """
    return not not re.match('^\w+$', s)

def valid_identifier(s):
    """

    >>> valid_identifier('hello')
    'hello'
    >>> valid_identifier('hello world')
    'hello_world'
    >>> valid_identifier('hello.world')
    'hello_world'
    >>> valid_identifier('hello-world')
    'hello_world'
    >>> print(valid_identifier(None))
    None
    """
    if isinstance(s, _strtypes):
        return s.replace(' ', '_').replace('.', '_').replace('-', '_')
    return s


class Expr(Node):
    """
    Symbolic expression of a computation

    All Blaze expressions (Join, By, Sort, ...) descend from this class.  It
    contains shared logic and syntax.  It in turn inherits from ``Node`` which
    holds all tree traversal logic
    """
    def _get_field(self, fieldname):
        if not isinstance(self.dshape.measure, Record):
            if fieldname == self._name:
                return self
            raise ValueError("Can not get field '%s' of non-record expression %s"
                    % (fieldname, self))
        return Field(self, fieldname)

    def __getitem__(self, key):
        if isinstance(key, _strtypes) and key in self.fields:
            return self._get_field(key)
        elif isinstance(key, Expr) and iscollection(key.dshape):
            return selection(self, key)
        elif (isinstance(key, list)
                and builtins.all(isinstance(k, _strtypes) for k in key)):
            if set(key).issubset(self.fields):
                return self._project(key)
            else:
                raise ValueError('Names %s not consistent with known names %s'
                        % (key, self.fields))
        elif (isinstance(key, tuple)
                and all(isinstance(k, (int, slice)) for k in key)):
            return Slice(self, key)
        elif isinstance(key, (slice, int)):
            return Slice(self, (key,))
        raise ValueError("Not understood %s[%s]" % (self, key))

    def map(self, func, schema=None, name=None):
        return Map(self, func, schema, name)

    def _project(self, key):
        return projection(self, key)

    @property
    def schema(self):
        return datashape.dshape(self.dshape.measure)

    @property
    def fields(self):
        if isinstance(self.dshape.measure, Record):
            return self.dshape.measure.names
        if hasattr(self, '_name'):
            return [self._name]

    def _len(self):
        try:
            return int(self.dshape[0])
        except TypeError:
            raise ValueError('Can not determine length of table with the '
                    'following datashape: %s' % self.dshape)

    def __len__(self): # pragma: no cover
        return self._len()

    def __iter__(self):
        raise NotImplementedError(
                'Iteration over expressions is not supported.\n'
                'Iterate over computed result instead, e.g. \n'
                "\titer(expr)           # don't do this\n"
                "\titer(compute(expr))  # do this instead")

    def __dir__(self):
        result = dir(type(self))
        if isrecord(self.dshape.measure) and self.fields:
            result.extend(list(map(valid_identifier, self.fields)))

        d = toolz.merge(schema_methods(self.dshape.measure),
                        dshape_methods(self.dshape))
        result.extend(list(d))

        return sorted(set(filter(isvalid_identifier, result)))

    def __getattr__(self, key):
        try:
            return object.__getattribute__(self, key)
        except AttributeError:
            fields = dict(zip(map(valid_identifier, self.fields),
                              self.fields))
            if self.fields and key in fields:
                if isscalar(self.dshape.measure): # t.foo.foo is t.foo
                    return self
                else:
                    return self[fields[key]]
            d = toolz.merge(schema_methods(self.dshape.measure),
                            dshape_methods(self.dshape))
            if key in d:
                func = d[key]
                if func in method_properties:
                    return func(self)
                else:
                    return functools.update_wrapper(partial(func, self), func)
            else:
                raise

    @property
    def _name(self):
        if (isscalar(self.dshape.measure) and
                len(self._inputs) == 1 and
                isscalar(self._child.dshape.measure)):
            return self._child._name


@memoize
def symbol(name, dshape, token=None):
    return Symbol(name, dshape, token=token)


class Symbol(Expr):
    """
    Symbolic data.  The leaf of a Blaze expression

    Example
    -------

    >>> points = symbol('points', '5 * 3 * {x: int, y: int}')<commit_msg>expr: cache Expr.__getattr__<commit_after>from __future__ import absolute_import, division, print_function

import toolz
import datashape
import functools
from toolz import concat, memoize, partial, pipe
from toolz.curried import map, filter
import re

from datashape import dshape, DataShape, Record, Var, Mono
from datashape.predicates import isscalar, iscollection, isboolean, isrecord

from ..compatibility import _strtypes, builtins
from .core import *
from .method_dispatch import select_functions
from ..dispatch import dispatch

__all__ = ['Expr', 'ElemWise', 'Field', 'Symbol', 'discover', 'Projection',
           'projection', 'Selection', 'selection', 'Label', 'label', 'Map',
           'ReLabel', 'relabel', 'Apply', 'Slice', 'shape', 'ndim', 'label',
           'symbol']


_attr_cache = dict()

def isvalid_identifier(s):
    """

    >>> isvalid_identifier('Hello')
    True
    >>> isvalid_identifier('Hello world')
    False
    >>> isvalid_identifier('Helloworld!')
    False
    """
    return not not re.match('^\w+$', s)

def valid_identifier(s):
    """

    >>> valid_identifier('hello')
    'hello'
    >>> valid_identifier('hello world')
    'hello_world'
    >>> valid_identifier('hello.world')
    'hello_world'
    >>> valid_identifier('hello-world')
    'hello_world'
    >>> print(valid_identifier(None))
    None
    """
    if isinstance(s, _strtypes):
        return s.replace(' ', '_').replace('.', '_').replace('-', '_')
    return s


class Expr(Node):
    """
    Symbolic expression of a computation

    All Blaze expressions (Join, By, Sort, ...) descend from this class.  It
    contains shared logic and syntax.  It in turn inherits from ``Node`` which
    holds all tree traversal logic
    """
    def _get_field(self, fieldname):
        if not isinstance(self.dshape.measure, Record):
            if fieldname == self._name:
                return self
            raise ValueError("Can not get field '%s' of non-record expression %s"
                    % (fieldname, self))
        return Field(self, fieldname)

    def __getitem__(self, key):
        if isinstance(key, _strtypes) and key in self.fields:
            return self._get_field(key)
        elif isinstance(key, Expr) and iscollection(key.dshape):
            return selection(self, key)
        elif (isinstance(key, list)
                and builtins.all(isinstance(k, _strtypes) for k in key)):
            if set(key).issubset(self.fields):
                return self._project(key)
            else:
                raise ValueError('Names %s not consistent with known names %s'
                        % (key, self.fields))
        elif (isinstance(key, tuple)
                and all(isinstance(k, (int, slice)) for k in key)):
            return Slice(self, key)
        elif isinstance(key, (slice, int)):
            return Slice(self, (key,))
        raise ValueError("Not understood %s[%s]" % (self, key))

    def map(self, func, schema=None, name=None):
        return Map(self, func, schema, name)

    def _project(self, key):
        return projection(self, key)

    @property
    def schema(self):
        return datashape.dshape(self.dshape.measure)

    @property
    def fields(self):
        if isinstance(self.dshape.measure, Record):
            return self.dshape.measure.names
        if hasattr(self, '_name'):
            return [self._name]

    def _len(self):
        try:
            return int(self.dshape[0])
        except TypeError:
            raise ValueError('Can not determine length of table with the '
                    'following datashape: %s' % self.dshape)

    def __len__(self): # pragma: no cover
        return self._len()

    def __iter__(self):
        raise NotImplementedError(
                'Iteration over expressions is not supported.\n'
                'Iterate over computed result instead, e.g. \n'
                "\titer(expr)           # don't do this\n"
                "\titer(compute(expr))  # do this instead")

    def __dir__(self):
        result = dir(type(self))
        if isrecord(self.dshape.measure) and self.fields:
            result.extend(list(map(valid_identifier, self.fields)))

        d = toolz.merge(schema_methods(self.dshape.measure),
                        dshape_methods(self.dshape))
        result.extend(list(d))

        return sorted(set(filter(isvalid_identifier, result)))

    def __getattr__(self, key):
        if key == '_hash':
            raise AttributeError()
        try:
            return _attr_cache[(self, key)]
        except:
            pass
        try:
            result = object.__getattribute__(self, key)
        except AttributeError:
            fields = dict(zip(map(valid_identifier, self.fields),
                              self.fields))
            if self.fields and key in fields:
                if isscalar(self.dshape.measure): # t.foo.foo is t.foo
                    result = self
                else:
                    result = self[fields[key]]
            else:
                d = toolz.merge(schema_methods(self.dshape.measure),
                                dshape_methods(self.dshape))
                if key in d:
                    func = d[key]
                    if func in method_properties:
                        result = func(self)
                    else:
                        result = functools.update_wrapper(partial(func, self), func)
                else:
                    raise
        _attr_cache[(self, key)] = result
        return result

    @property
    def _name(self):
        if (isscalar(self.dshape.measure) and
                len(self._inputs) == 1 and
                isscalar(self._child.dshape.measure)):
            return self._child._name


@memoize
def symbol(name, dshape, token=None):
    return Symbol(name, dshape, token=token)


class Symbol(Expr):
    """
    Symbolic data.  The leaf of a Blaze expression

    Example
    -------

    >>> points = symbol('points', '5 * 3 * {x: int, y: int}')
{'fraction_non_alphanumeric': 0.11422469030328919, 'fraction_numerical': 0.0006834686031610423, 'mean_word_length': 2.1116427432216907, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4479288', 'n_tokens_mistral': 3333, 'n_tokens_neox': 3234, 'n_words': 906}