<commit_before>"""
Contains all classes and functions to deal with lists, dicts, generators and
iterators in general.

Array modifications
*******************

If the content of an array (``set``/``list``) is requested somewhere, the
current module will be checked for appearances of ``arr.append``,
``arr.insert``, etc.  If the ``arr`` name points to an actual array, the
content will be added

This can be really cpu intensive, as you can imagine. Because |jedi| has to
follow **every** ``append`` and check wheter it's the right array. However this
works pretty good, because in *slow* cases, the recursion detector and other
settings will stop this process.

It is important to note that:

1. Array modfications work only in the current module.
2. Jedi only checks Array additions; ``list.pop``, etc are ignored.
"""
from itertools import chain

from jedi import common
from jedi import debug
from jedi import settings
from jedi._compatibility import use_metaclass, is_py3, unicode
from jedi.parser import representation as pr
from jedi.evaluate import compiled
from jedi.evaluate import helpers
from jedi.evaluate import precedence
from jedi.evaluate.cache import CachedMetaClass, memoize_default, NO_DEFAULT
from jedi.cache import underscore_memoization
from jedi.evaluate import analysis


class IterableWrapper(pr.Base):
    def is_class(self):
        return False


class GeneratorMixin(object):
    @underscore_memoization
    def _get_defined_names(self):
        """
        Returns a list of names that define a generator, which can return the
        content of a generator.
        """
        executes_generator = '__next__', 'send', 'next'
        for name in compiled.generator_obj.get_defined_names():
            if name.name in executes_generator:
                parent = GeneratorMethod(self, name.parent)
                yield helpers.FakeName(name.name, parent)
            else:
                yield name

    def scope_names_generator(self, position=None):
        yield self, self._get_defined_names()


class Generator(use_metaclass(CachedMetaClass, IterableWrapper, GeneratorMixin)):
    """Handling of `yield` functions."""
    def __init__(self, evaluator, func, var_args):
        super(Generator, self).__init__()
        self._evaluator = evaluator
        self.func = func
        self.var_args = var_args

    def iter_content(self):
        """ returns the content of __iter__ """
        # Directly execute it, because with a normal call to py__call__ a
        # Generator will be returned.
        from jedi.evaluate.representation import FunctionExecution
        return FunctionExecution(self._evaluator, self.func, self.var_args).get_return_types()

    def get_index_types(self, index_array):
        #debug.warning('Tried to get array access on a generator: %s', self)
        analysis.add(self._evaluator, 'type-error-generator', index_array)
        return []

    def get_exact_index_types(self, index):
        """
        Exact lookups are used for tuple lookups, which are perfectly fine if
        used with generators.
        """
        return [self.iter_content()[index]]

    def __getattr__(self, name):
        if name not in ['start_pos', 'end_pos', 'parent', 'get_imports',
                        'asserts', 'doc', 'docstr', 'get_parent_until',
                        'get_code', 'subscopes']:
            raise AttributeError("Accessing %s of %s is not allowed."
                                 % (self, name))
        return getattr(self.func, name)

    def __repr__(self):
        return "<%s of %s>" % (type(self).__name__, self.func)


class GeneratorMethod(IterableWrapper):
    """``__next__`` and ``send`` methods."""
    def __init__(self, generator, builtin_func):
        self._builtin_func = builtin_func
        self._generator = generator

    def py__call__(self, evaluator, params):
        # TODO add TypeError if params are given.
        return self._generator.iter_content()

    def __getattr__(self, name):
        return getattr(self._builtin_func, name)


class Comprehension(IterableWrapper):
    @staticmethod
    def from_atom(evaluator, atom):
        mapping = {
            '(': GeneratorComprehension,
            '[': ListComprehension
        }
        return mapping[atom.children[0]](evaluator, atom)

    def __init__(self, evaluator, atom):
        self._evaluator = evaluator
        self._atom = atom

    @memoize_default()
    def eval_node(self):
        """
        The first part `x + 1` of the list comprehension:

            [x + 1 for x in foo]
        """
        comprehension = self._atom.children[1]
        # For nested comprehensions we need to search the last one.
        last = comprehension.children[-1]
        last_comp = comprehension.children[1]
        while True:
            if isinstance(last, pr.CompFor):
                last_comp = last
            elif not pr.is_node(last, 'comp_if'):
                break
            last = last.children[-1]

        return helpers.deep_ast_copy(comprehension.children[0], {comprehension: last_comp})

    def __repr__(self):
        return "<e%s of %s>" % (type(self).__name__, self._atom)


class ListComprehension(Comprehension):
    def get_index_types(self, evaluator, index):
        return self._evaluator.eval_element(self.eval_node())



class GeneratorComprehension(Comprehension, GeneratorMixin):
    def iter_content(self):
        return self._evaluator.eval_statement_element(self.comprehension)


class Array(IterableWrapper):
    """
    Used as a mirror to pr.Array, if needed. It defines some getter
    methods which are important in this module.
    """
    mapping = {'(': pr.Array.TUPLE,
               '[': pr.Array.LIST,
               '{': pr.Array.DICT}

    def __init__(self, evaluator, atom):<commit_msg>Make Generator tuple assignments work.<commit_after>"""
Contains all classes and functions to deal with lists, dicts, generators and
iterators in general.

Array modifications
*******************

If the content of an array (``set``/``list``) is requested somewhere, the
current module will be checked for appearances of ``arr.append``,
``arr.insert``, etc.  If the ``arr`` name points to an actual array, the
content will be added

This can be really cpu intensive, as you can imagine. Because |jedi| has to
follow **every** ``append`` and check wheter it's the right array. However this
works pretty good, because in *slow* cases, the recursion detector and other
settings will stop this process.

It is important to note that:

1. Array modfications work only in the current module.
2. Jedi only checks Array additions; ``list.pop``, etc are ignored.
"""
from itertools import chain

from jedi import common
from jedi import debug
from jedi import settings
from jedi._compatibility import use_metaclass, is_py3, unicode
from jedi.parser import representation as pr
from jedi.evaluate import compiled
from jedi.evaluate import helpers
from jedi.evaluate import precedence
from jedi.evaluate.cache import CachedMetaClass, memoize_default, NO_DEFAULT
from jedi.cache import underscore_memoization
from jedi.evaluate import analysis


class IterableWrapper(pr.Base):
    def is_class(self):
        return False


class GeneratorMixin(object):
    @underscore_memoization
    def _get_defined_names(self):
        """
        Returns a list of names that define a generator, which can return the
        content of a generator.
        """
        executes_generator = '__next__', 'send', 'next'
        for name in compiled.generator_obj.get_defined_names():
            if name.name in executes_generator:
                parent = GeneratorMethod(self, name.parent)
                yield helpers.FakeName(name.name, parent)
            else:
                yield name

    def scope_names_generator(self, position=None):
        yield self, self._get_defined_names()

    def get_index_types(self, evaluator, index_array):
        #debug.warning('Tried to get array access on a generator: %s', self)
        analysis.add(self._evaluator, 'type-error-generator', index_array)
        return []

    def get_exact_index_types(self, index):
        """
        Exact lookups are used for tuple lookups, which are perfectly fine if
        used with generators.
        """
        return [self.iter_content()[index]]



class Generator(use_metaclass(CachedMetaClass, IterableWrapper, GeneratorMixin)):
    """Handling of `yield` functions."""
    def __init__(self, evaluator, func, var_args):
        super(Generator, self).__init__()
        self._evaluator = evaluator
        self.func = func
        self.var_args = var_args

    def iter_content(self):
        """ returns the content of __iter__ """
        # Directly execute it, because with a normal call to py__call__ a
        # Generator will be returned.
        from jedi.evaluate.representation import FunctionExecution
        return FunctionExecution(self._evaluator, self.func, self.var_args).get_return_types()

    def __getattr__(self, name):
        if name not in ['start_pos', 'end_pos', 'parent', 'get_imports',
                        'asserts', 'doc', 'docstr', 'get_parent_until',
                        'get_code', 'subscopes']:
            raise AttributeError("Accessing %s of %s is not allowed."
                                 % (self, name))
        return getattr(self.func, name)

    def __repr__(self):
        return "<%s of %s>" % (type(self).__name__, self.func)


class GeneratorMethod(IterableWrapper):
    """``__next__`` and ``send`` methods."""
    def __init__(self, generator, builtin_func):
        self._builtin_func = builtin_func
        self._generator = generator

    def py__call__(self, evaluator, params):
        # TODO add TypeError if params are given.
        return self._generator.iter_content()

    def __getattr__(self, name):
        return getattr(self._builtin_func, name)


class Comprehension(IterableWrapper):
    @staticmethod
    def from_atom(evaluator, atom):
        mapping = {
            '(': GeneratorComprehension,
            '[': ListComprehension
        }
        return mapping[atom.children[0]](evaluator, atom)

    def __init__(self, evaluator, atom):
        self._evaluator = evaluator
        self._atom = atom

    @memoize_default()
    def eval_node(self):
        """
        The first part `x + 1` of the list comprehension:

            [x + 1 for x in foo]
        """
        comprehension = self._atom.children[1]
        # For nested comprehensions we need to search the last one.
        last = comprehension.children[-1]
        last_comp = comprehension.children[1]
        while True:
            if isinstance(last, pr.CompFor):
                last_comp = last
            elif not pr.is_node(last, 'comp_if'):
                break
            last = last.children[-1]

        return helpers.deep_ast_copy(comprehension.children[0], {comprehension: last_comp})

    def __repr__(self):
        return "<e%s of %s>" % (type(self).__name__, self._atom)


class ListComprehension(Comprehension):
    def get_index_types(self, evaluator, index):
        return self._evaluator.eval_element(self.eval_node())



class GeneratorComprehension(Comprehension, GeneratorMixin):
    def iter_content(self):
        return self._evaluator.eval_element(self.eval_node())


class Array(IterableWrapper):
    """
    Used as a mirror to pr.Array, if needed. It defines some getter
    methods which are important in this module.
    """
    mapping = {'(': pr.Array.TUPLE,
               '[': pr.Array.LIST,
               '{': pr.Array.DICT}

    def __init__(self, evaluator, atom):
{'fraction_non_alphanumeric': 0.09758611803109699, 'fraction_numerical': 0.0018898720041233572, 'mean_word_length': 2.860079575596817, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2100545', 'n_tokens_mistral': 3331, 'n_tokens_neox': 3224, 'n_words': 1032}