<commit_before>"""
The API basically only provides one class. You can create a :class:`Script` and
use its methods.

Additionally you can add a debug function with :func:`set_debug_function` and
catch :exc:`NotFoundError` which is being raised if your completion is not
possible.

.. warning:: Please, note that Jedi is **not thread safe**.
"""
import re
import os
import warnings
import sys
from itertools import chain

from jedi._compatibility import next, unicode, builtins
from jedi.parser import Parser, load_grammar
from jedi.parser.tokenize import source_tokens
from jedi.parser import tree as pr
from jedi.parser.user_context import UserContext, UserContextParser
from jedi import debug
from jedi import settings
from jedi import common
from jedi import cache
from jedi.api import keywords
from jedi.api import classes
from jedi.api import interpreter
from jedi.api import usages
from jedi.api import helpers
from jedi.evaluate import Evaluator
from jedi.evaluate import representation as er
from jedi.evaluate import compiled
from jedi.evaluate import imports
from jedi.evaluate.cache import memoize_default
from jedi.evaluate.helpers import FakeName, get_module_name_parts
from jedi.evaluate.finder import get_names_of_scope, filter_private_variable
from jedi.evaluate.helpers import search_call_signatures
from jedi.evaluate import analysis

# Jedi uses lots and lots of recursion. By setting this a little bit higher, we
# can remove some "maximum recursion depth" errors.
sys.setrecursionlimit(2000)


class NotFoundError(Exception):
    """A custom error to avoid catching the wrong exceptions."""
    # TODO deprecate this.


class Script(object):
    """
    A Script is the base for completions, goto or whatever you want to do with
    |jedi|.

    You can either use the ``source`` parameter or ``path`` to read a file.
    Usually you're going to want to use both of them (in an editor).

    :param source: The source code of the current file, separated by newlines.
    :type source: str
    :param line: The line to perform actions on (starting with 1).
    :type line: int
    :param col: The column of the cursor (starting with 0).
    :type col: int
    :param path: The path of the file in the file system, or ``''`` if
        it hasn't been saved yet.
    :type path: str or None
    :param encoding: The encoding of ``source``, if it is not a
        ``unicode`` object (default ``'utf-8'``).
    :type encoding: str
    :param source_encoding: The encoding of ``source``, if it is not a
        ``unicode`` object (default ``'utf-8'``).
    :type encoding: str
    """
    def __init__(self, source=None, line=None, column=None, path=None,
                 encoding='utf-8', source_path=None, source_encoding=None):
        if source_path is not None:
            warnings.warn("Use path instead of source_path.", DeprecationWarning)
            path = source_path
        if source_encoding is not None:
            warnings.warn("Use encoding instead of source_encoding.", DeprecationWarning)
            encoding = source_encoding

        self._orig_path = path
        self.path = None if path is None else os.path.abspath(path)

        if source is None:
            with open(path) as f:
                source = f.read()

        self.source = common.source_to_unicode(source, encoding)
        lines = common.splitlines(self.source)
        line = max(len(lines), 1) if line is None else line
        if not (0 < line <= len(lines)):
            raise ValueError('`line` parameter is not in a valid range.')

        line_len = len(lines[line - 1])
        column = line_len if column is None else column
        if not (0 <= column <= line_len):
            raise ValueError('`column` parameter is not in a valid range.')
        self._pos = line, column

        cache.clear_time_caches()
        debug.reset_time()
        self._grammar = load_grammar('grammar3.4')
        self._user_context = UserContext(self.source, self._pos)
        self._parser = UserContextParser(self._grammar, self.source, path,
                                         self._pos, self._user_context)
        self._evaluator = Evaluator(self._grammar)
        debug.speed('init')

    @property
    def source_path(self):
        """
        .. deprecated:: 0.7.0
           Use :attr:`.path` instead.
        .. todo:: Remove!
        """
        warnings.warn("Use path instead of source_path.", DeprecationWarning)
        return self.path

    def __repr__(self):
        return '<%s: %s>' % (self.__class__.__name__, repr(self._orig_path))

    def completions(self):
        """
        Return :class:`classes.Completion` objects. Those objects contain
        information about the completions, more than just names.

        :return: Completion objects, sorted by name and __ comes last.
        :rtype: list of :class:`classes.Completion`
        """
        def get_completions(user_stmt, bs):
            if user_stmt is None:
                module = self._parser.module()
                importer = helpers.check_error_statements(
                    self._evaluator, module, self._pos
                )
                #print(importer.completion_names(self._evaluator, True))
                if importer is not None:
                    names = importer.completion_names(self._evaluator, True)
                    return [(name, module) for name in names]
            elif isinstance(user_stmt, pr.Import):
                # TODO this paragraph is necessary, but not sure it works.
                context = self._user_context.get_context()
                next(context)  # skip the path
                if next(context) == 'from':
                    # completion is just "import" if before stands from ..
                    return ((k, bs) for k in keywords.keyword_names('import'))

                module = self._parser.module()
                name = user_stmt.name_for_position(self._pos)
                imp = imports.ImportWrapper(self._evaluator, name)
                return [(n, module) for n in imp.get_defined_names()]

            return self._simple_complete(path, dot, like)

        def completion_possible(path):
            """
            The completion logic is kind of complicated, because we strip the
            last word part. To ignore certain strange patterns with dots, just
            use regex.
            """

            if re.match('\d+\.\.$|\.{4}$', path):
                return True  # check Ellipsis and float literal `1.`

            return not re.search(r'^\.|^\d\.$|\.\.$', path)

        debug.speed('completions start')
        path = self._user_context.get_path_until_cursor()
        # TODO still needed with the new parser?
        #if not completion_possible(path):
        #    return []
        path, dot, like = helpers.completion_parts(path)

        user_stmt = self._parser.user_stmt_with_whitespace()

        b = compiled.builtin
        completions = get_completions(user_stmt, b)

        if not dot:
            # add named params
            for call_sig in self.call_signatures():
                # Allow protected access, because it's a public API.
                module = call_sig._name.get_parent_until()
                # Compiled modules typically don't allow keyword arguments.
                if not isinstance(module, compiled.CompiledObject):
                    for p in call_sig.params:
                        # Allow access on _definition here, because it's a
                        # public API and we don't want to make the internal
                        # Name object public.
                        if p._definition.stars == 0:  # no *args/**kwargs
                            completions.append((p._name, p._name))

            if not path and not isinstance(user_stmt, pr.Import):
                # add keywords
                #completions += ((k, b) for k in keywords.keyword_names(all=True))
                pass
                # TODO delete? We should search for valid parser
                # transformations.

        needs_dot = not dot and path

        comps = []
        comp_dct = {}
        for c, s in set(completions):
            n = str(c)
            if settings.case_insensitive_completion \
                    and n.lower().startswith(like.lower()) \
                    or n.startswith(like):
                if not filter_private_variable(s, user_stmt or self._parser.user_scope(), n):
                    if isinstance(c.parent, (pr.Function, pr.Class)):
                        # TODO I think this is a hack. It should be an
                        #   er.Function/er.Class before that.
                        c = er.wrap(self._evaluator, c.parent).name
                    new = classes.Completion(self._evaluator, c, needs_dot, len(like), s)
                    k = (new.name, new.complete)  # key
                    if k in comp_dct and settings.no_completion_duplicates:
                        comp_dct[k]._same_name_completions.append(new)
                    else:
                        comp_dct[k] = new
                        comps.append(new)

        debug.speed('completions end')

        return sorted(comps, key=lambda x: (x.name.startswith('__'),
                                            x.name.startswith('_'),
                                            x.name.lower()))

    def _simple_complete(self, path, dot, like):
        if not path and not dot:
            scope_names_generator = get_names_of_scope(self._evaluator,
                                                       self._parser.user_scope(),
                                                       self._pos)
            completions = []
            for scope, name_list in scope_names_generator:
                for c in name_list:
                    completions.append((c, scope))
        elif self._get_under_cursor_stmt(path) is None:
            return []
        else:
            scopes = list(self._prepare_goto(path, True))
            completions = []
            debug.dbg('possible completion scopes: %s', scopes)
            for s in scopes:
                if s.isinstance(er.Function):
                    names = s.get_magic_function_names()
                elif isinstance(s, imports.ImportWrapper):
                    under = like + self._user_context.get_path_after_cursor()
                    if under == 'import':
                        current_line = self._user_context.get_position_line()
                        if not current_line.endswith('import import'):
                            continue
                    """
                    a = s.import_stmt.alias
                    if a and a.start_pos <= self._pos <= a.end_pos:
                        continue
                    """
                    # TODO what to do with this?
                    names = s.get_defined_names(on_import_stmt=True)
                else:
                    names = []
                    for _, new_names in s.scope_names_generator():
                        names += new_names

                for c in names:
                    completions.append((c, s))
        return completions

    def _prepare_goto(self, goto_path, is_completion=False):
        """
        Base for completions/goto. Basically it returns the resolved scopes
        under cursor.
        """
        debug.dbg('start: %s in %s', goto_path, self._parser.user_scope())

        user_stmt = self._parser.user_stmt_with_whitespace()
        if not user_stmt and len(goto_path.split('\n')) > 1:
            # If the user_stmt is not defined and the goto_path is multi line,
            # something's strange. Most probably the backwards tokenizer
            # matched to much.
            return []

        if isinstance(user_stmt, pr.Import):
            scopes = [helpers.get_on_import_stmt(self._evaluator, self._user_context,
                                                 user_stmt, is_completion)[0]]
        else:
            # just parse one statement, take it and evaluate it
            eval_stmt = self._get_under_cursor_stmt(goto_path)
            if eval_stmt is None:
                return []

            if not is_completion:
                # goto_definition returns definitions of its statements if the
                # cursor is on the assignee. By changing the start_pos of our
                # "pseudo" statement, the Jedi evaluator can find the assignees.
                
                # TODO remove?
                if False and user_stmt is not None:
                    eval_stmt.start_pos = user_stmt.end_pos
            scopes = self._evaluator.eval_statement(eval_stmt)

        return scopes

    @memoize_default()
    def _get_under_cursor_stmt(self, cursor_txt):
        tokenizer = source_tokens(cursor_txt, line_offset=self._pos[0] - 1)
        r = Parser(self._grammar, cursor_txt, tokenizer=tokenizer)
        try:
            # Take the last statement available.
            stmt = r.module.statements[-1]
        except IndexError:
            return None
            raise NotFoundError()
        if not isinstance(stmt, (pr.ExprStmt, pr.KeywordStatement)):
            raise NotImplementedError
            raise NotFoundError()

        # TODO remove?
        user_stmt = self._parser.user_stmt()
        if user_stmt is None:
            # Set the start_pos to a pseudo position, that doesn't exist but works
            # perfectly well (for both completions in docstrings and statements).
            pos = self._pos
        else:
            pos = user_stmt.start_pos

        #stmt.move(pos[0] - 1, pos[1])
        stmt.parent = self._parser.user_scope()
        return stmt

    def complete(self):<commit_msg>Fix positioning of the user statements.<commit_after>"""
The API basically only provides one class. You can create a :class:`Script` and
use its methods.

Additionally you can add a debug function with :func:`set_debug_function` and
catch :exc:`NotFoundError` which is being raised if your completion is not
possible.

.. warning:: Please, note that Jedi is **not thread safe**.
"""
import re
import os
import warnings
import sys
from itertools import chain

from jedi._compatibility import next, unicode, builtins
from jedi.parser import Parser, load_grammar
from jedi.parser.tokenize import source_tokens
from jedi.parser import tree as pr
from jedi.parser.user_context import UserContext, UserContextParser
from jedi import debug
from jedi import settings
from jedi import common
from jedi import cache
from jedi.api import keywords
from jedi.api import classes
from jedi.api import interpreter
from jedi.api import usages
from jedi.api import helpers
from jedi.evaluate import Evaluator
from jedi.evaluate import representation as er
from jedi.evaluate import compiled
from jedi.evaluate import imports
from jedi.evaluate.cache import memoize_default
from jedi.evaluate.helpers import FakeName, get_module_name_parts
from jedi.evaluate.finder import get_names_of_scope, filter_private_variable
from jedi.evaluate.helpers import search_call_signatures
from jedi.evaluate import analysis

# Jedi uses lots and lots of recursion. By setting this a little bit higher, we
# can remove some "maximum recursion depth" errors.
sys.setrecursionlimit(2000)


class NotFoundError(Exception):
    """A custom error to avoid catching the wrong exceptions."""
    # TODO deprecate this.


class Script(object):
    """
    A Script is the base for completions, goto or whatever you want to do with
    |jedi|.

    You can either use the ``source`` parameter or ``path`` to read a file.
    Usually you're going to want to use both of them (in an editor).

    :param source: The source code of the current file, separated by newlines.
    :type source: str
    :param line: The line to perform actions on (starting with 1).
    :type line: int
    :param col: The column of the cursor (starting with 0).
    :type col: int
    :param path: The path of the file in the file system, or ``''`` if
        it hasn't been saved yet.
    :type path: str or None
    :param encoding: The encoding of ``source``, if it is not a
        ``unicode`` object (default ``'utf-8'``).
    :type encoding: str
    :param source_encoding: The encoding of ``source``, if it is not a
        ``unicode`` object (default ``'utf-8'``).
    :type encoding: str
    """
    def __init__(self, source=None, line=None, column=None, path=None,
                 encoding='utf-8', source_path=None, source_encoding=None):
        if source_path is not None:
            warnings.warn("Use path instead of source_path.", DeprecationWarning)
            path = source_path
        if source_encoding is not None:
            warnings.warn("Use encoding instead of source_encoding.", DeprecationWarning)
            encoding = source_encoding

        self._orig_path = path
        self.path = None if path is None else os.path.abspath(path)

        if source is None:
            with open(path) as f:
                source = f.read()

        self.source = common.source_to_unicode(source, encoding)
        lines = common.splitlines(self.source)
        line = max(len(lines), 1) if line is None else line
        if not (0 < line <= len(lines)):
            raise ValueError('`line` parameter is not in a valid range.')

        line_len = len(lines[line - 1])
        column = line_len if column is None else column
        if not (0 <= column <= line_len):
            raise ValueError('`column` parameter is not in a valid range.')
        self._pos = line, column

        cache.clear_time_caches()
        debug.reset_time()
        self._grammar = load_grammar('grammar3.4')
        self._user_context = UserContext(self.source, self._pos)
        self._parser = UserContextParser(self._grammar, self.source, path,
                                         self._pos, self._user_context)
        self._evaluator = Evaluator(self._grammar)
        debug.speed('init')

    @property
    def source_path(self):
        """
        .. deprecated:: 0.7.0
           Use :attr:`.path` instead.
        .. todo:: Remove!
        """
        warnings.warn("Use path instead of source_path.", DeprecationWarning)
        return self.path

    def __repr__(self):
        return '<%s: %s>' % (self.__class__.__name__, repr(self._orig_path))

    def completions(self):
        """
        Return :class:`classes.Completion` objects. Those objects contain
        information about the completions, more than just names.

        :return: Completion objects, sorted by name and __ comes last.
        :rtype: list of :class:`classes.Completion`
        """
        def get_completions(user_stmt, bs):
            if user_stmt is None:
                module = self._parser.module()
                importer = helpers.check_error_statements(
                    self._evaluator, module, self._pos
                )
                #print(importer.completion_names(self._evaluator, True))
                if importer is not None:
                    names = importer.completion_names(self._evaluator, True)
                    return [(name, module) for name in names]
            elif isinstance(user_stmt, pr.Import):
                # TODO this paragraph is necessary, but not sure it works.
                context = self._user_context.get_context()
                next(context)  # skip the path
                if next(context) == 'from':
                    # completion is just "import" if before stands from ..
                    return ((k, bs) for k in keywords.keyword_names('import'))

                module = self._parser.module()
                name = user_stmt.name_for_position(self._pos)
                imp = imports.ImportWrapper(self._evaluator, name)
                return [(n, module) for n in imp.get_defined_names()]

            return self._simple_complete(path, dot, like)

        def completion_possible(path):
            """
            The completion logic is kind of complicated, because we strip the
            last word part. To ignore certain strange patterns with dots, just
            use regex.
            """

            if re.match('\d+\.\.$|\.{4}$', path):
                return True  # check Ellipsis and float literal `1.`

            return not re.search(r'^\.|^\d\.$|\.\.$', path)

        debug.speed('completions start')
        path = self._user_context.get_path_until_cursor()
        # TODO still needed with the new parser?
        #if not completion_possible(path):
        #    return []
        path, dot, like = helpers.completion_parts(path)

        user_stmt = self._parser.user_stmt_with_whitespace()

        b = compiled.builtin
        completions = get_completions(user_stmt, b)

        if not dot:
            # add named params
            for call_sig in self.call_signatures():
                # Allow protected access, because it's a public API.
                module = call_sig._name.get_parent_until()
                # Compiled modules typically don't allow keyword arguments.
                if not isinstance(module, compiled.CompiledObject):
                    for p in call_sig.params:
                        # Allow access on _definition here, because it's a
                        # public API and we don't want to make the internal
                        # Name object public.
                        if p._definition.stars == 0:  # no *args/**kwargs
                            completions.append((p._name, p._name))

            if not path and not isinstance(user_stmt, pr.Import):
                # add keywords
                #completions += ((k, b) for k in keywords.keyword_names(all=True))
                pass
                # TODO delete? We should search for valid parser
                # transformations.

        needs_dot = not dot and path

        comps = []
        comp_dct = {}
        for c, s in set(completions):
            n = str(c)
            if settings.case_insensitive_completion \
                    and n.lower().startswith(like.lower()) \
                    or n.startswith(like):
                if not filter_private_variable(s, user_stmt or self._parser.user_scope(), n):
                    if isinstance(c.parent, (pr.Function, pr.Class)):
                        # TODO I think this is a hack. It should be an
                        #   er.Function/er.Class before that.
                        c = er.wrap(self._evaluator, c.parent).name
                    new = classes.Completion(self._evaluator, c, needs_dot, len(like), s)
                    k = (new.name, new.complete)  # key
                    if k in comp_dct and settings.no_completion_duplicates:
                        comp_dct[k]._same_name_completions.append(new)
                    else:
                        comp_dct[k] = new
                        comps.append(new)

        debug.speed('completions end')

        return sorted(comps, key=lambda x: (x.name.startswith('__'),
                                            x.name.startswith('_'),
                                            x.name.lower()))

    def _simple_complete(self, path, dot, like):
        if not path and not dot:
            scope_names_generator = get_names_of_scope(self._evaluator,
                                                       self._parser.user_scope(),
                                                       self._pos)
            completions = []
            for scope, name_list in scope_names_generator:
                for c in name_list:
                    completions.append((c, scope))
        elif self._get_under_cursor_stmt(path) is None:
            return []
        else:
            scopes = list(self._prepare_goto(path, True))
            completions = []
            debug.dbg('possible completion scopes: %s', scopes)
            for s in scopes:
                if s.isinstance(er.Function):
                    names = s.get_magic_function_names()
                elif isinstance(s, imports.ImportWrapper):
                    under = like + self._user_context.get_path_after_cursor()
                    if under == 'import':
                        current_line = self._user_context.get_position_line()
                        if not current_line.endswith('import import'):
                            continue
                    """
                    a = s.import_stmt.alias
                    if a and a.start_pos <= self._pos <= a.end_pos:
                        continue
                    """
                    # TODO what to do with this?
                    names = s.get_defined_names(on_import_stmt=True)
                else:
                    names = []
                    for _, new_names in s.scope_names_generator():
                        names += new_names

                for c in names:
                    completions.append((c, s))
        return completions

    def _prepare_goto(self, goto_path, is_completion=False):
        """
        Base for completions/goto. Basically it returns the resolved scopes
        under cursor.
        """
        debug.dbg('start: %s in %s', goto_path, self._parser.user_scope())

        user_stmt = self._parser.user_stmt_with_whitespace()
        if not user_stmt and len(goto_path.split('\n')) > 1:
            # If the user_stmt is not defined and the goto_path is multi line,
            # something's strange. Most probably the backwards tokenizer
            # matched to much.
            return []

        if isinstance(user_stmt, pr.Import):
            scopes = [helpers.get_on_import_stmt(self._evaluator, self._user_context,
                                                 user_stmt, is_completion)[0]]
        else:
            # just parse one statement, take it and evaluate it
            eval_stmt = self._get_under_cursor_stmt(goto_path)
            if eval_stmt is None:
                return []

            if not is_completion:
                # goto_definition returns definitions of its statements if the
                # cursor is on the assignee. By changing the start_pos of our
                # "pseudo" statement, the Jedi evaluator can find the assignees.
                
                # TODO remove?
                if False and user_stmt is not None:
                    eval_stmt.start_pos = user_stmt.end_pos
            scopes = self._evaluator.eval_statement(eval_stmt)

        return scopes

    @memoize_default()
    def _get_under_cursor_stmt(self, cursor_txt):
        tokenizer = source_tokens(cursor_txt, line_offset=self._pos[0] - 1)
        r = Parser(self._grammar, cursor_txt, tokenizer=tokenizer)
        try:
            # Take the last statement available.
            stmt = r.module.statements[-1]
        except IndexError:
            return None
            raise NotFoundError()
        if not isinstance(stmt, (pr.ExprStmt, pr.KeywordStatement)):
            raise NotImplementedError
            raise NotFoundError()

        user_stmt = self._parser.user_stmt()
        if user_stmt is None:
            # Set the start_pos to a pseudo position, that doesn't exist but works
            # perfectly well (for both completions in docstrings and statements).
            pos = self._pos
        else:
            pos = user_stmt.start_pos

        stmt.move(0, pos[1])
        stmt.parent = self._parser.user_scope()
        return stmt

    def complete(self):
{'fraction_non_alphanumeric': 0.08336071858911162, 'fraction_numerical': 0.0020812794391499617, 'mean_word_length': 1.8484659386375455, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6846285', 'n_tokens_mistral': 7291, 'n_tokens_neox': 6951, 'n_words': 2291}