<commit_before># copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
# contact http://www.logilab.fr/ -- mailto:<EMAIL>
#
# This file is part of astroid.
#
# astroid is free software: you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by the
# Free Software Foundation, either version 2.1 of the License, or (at your
# option) any later version.
#
# astroid is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License
# for more details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with astroid. If not, see <http://www.gnu.org/licenses/>.
"""astroid manager: avoid multiple astroid build of a same module when
possible by providing a class responsible to get astroid representation
from various source and using a cache of built modules)
"""
from __future__ import print_function

__docformat__ = "restructuredtext en"

import collections
import imp
import os
from os.path import dirname, join, isdir, exists
from warnings import warn
import zipimport

from logilab.common.configuration import OptionsProviderMixIn

from astroid.exceptions import AstroidBuildingException
from astroid import modutils


def astroid_wrapper(func, modname):
    """wrapper to give to AstroidManager.project_from_files"""
    print('parsing %s...' % modname)
    try:
        return func(modname)
    except AstroidBuildingException as exc:
        print(exc)
    except Exception as exc:
        import traceback
        traceback.print_exc()

def _silent_no_wrap(func, modname):
    """silent wrapper that doesn't do anything; can be used for tests"""
    return func(modname)

def safe_repr(obj):
    try:
        return repr(obj)
    except:
        return '???'



class AstroidManager(OptionsProviderMixIn):
    """the astroid manager, responsible to build astroid from files
     or modules.

    Use the Borg pattern.
    """

    name = 'astroid loader'
    options = (("ignore",
                {'type' : "csv", 'metavar' : "<file>",
                 'dest' : "black_list", "default" : ('CVS',),
                 'help' : "add <file> (may be a directory) to the black list\
. It should be a base name, not a path. You may set this option multiple times\
."}),
               ("project",
                {'default': "No Name", 'type' : 'string', 'short': 'p',
                 'metavar' : '<project name>',
                 'help' : 'set the project name.'}),
              )
    brain = {}
    def __init__(self):
        self.__dict__ = AstroidManager.brain
        if not self.__dict__:
            OptionsProviderMixIn.__init__(self)
            self.load_defaults()
            # NOTE: cache entries are added by the [re]builder
            self.astroid_cache = {}
            self._mod_file_cache = {}
            self.transforms = collections.defaultdict(list)
            self._failed_import_hooks = []

    def ast_from_file(self, filepath, modname=None, fallback=True, source=False):
        """given a module name, return the astroid object"""
        try:
            filepath = modutils.get_source_file(filepath, include_no_ext=True)
            source = True
        except modutils.NoSourceFile:
            pass
        if modname is None:
            try:
                modname = '.'.join(modutils.modpath_from_file(filepath))
            except ImportError:
                modname = filepath
        if modname in self.astroid_cache and self.astroid_cache[modname].file == filepath:
            return self.astroid_cache[modname]
        if source:
            from astroid.builder import AstroidBuilder
            return AstroidBuilder(self).file_build(filepath, modname)
        elif fallback and modname:
            return self.ast_from_module_name(modname)
        raise AstroidBuildingException('unable to get astroid for file %s' %
                                       filepath)

    def _build_stub_module(self, modname):
        from astroid.builder import AstroidBuilder
        return AstroidBuilder(self).string_build('', modname)

    def ast_from_module_name(self, modname, context_file=None):
        """given a module name, return the astroid object"""
        if modname in self.astroid_cache:
            return self.astroid_cache[modname]
        if modname == '__main__':
            return self._build_stub_module(modname)
        old_cwd = os.getcwd()
        if context_file:
            os.chdir(dirname(context_file))
        try:
            filepath, mp_type = self.file_from_module_name(modname, context_file)
            if mp_type == modutils.PY_ZIPMODULE:
                module = self.zip_import_data(filepath)
                if module is not None:
                    return module
            elif mp_type in (imp.C_BUILTIN, imp.C_EXTENSION):
                if mp_type == imp.C_EXTENSION and not modutils.is_standard_module(modname):
                    return self._build_stub_module(modname)
                try:
                    module = modutils.load_module_from_name(modname)
                except Exception as ex:
                    msg = 'Unable to load module %s (%s)' % (modname, ex)
                    raise AstroidBuildingException(msg)
                return self.ast_from_module(module, modname)
            elif mp_type == imp.PY_COMPILED:
                raise AstroidBuildingException("Unable to load compiled module %s" % (modname,))
            if filepath is None:
                raise AstroidBuildingException("Unable to load module %s" % (modname,))
            return self.ast_from_file(filepath, modname, fallback=False)
        except AstroidBuildingException as e:
            for hook in self._failed_import_hooks:
                try:
                    return hook(modname)
                except AstroidBuildingException:
                    pass
            raise e
        finally:
            os.chdir(old_cwd)

    def zip_import_data(self, filepath):
        if zipimport is None:
            return None
        from astroid.builder import AstroidBuilder<commit_msg>Add support for more fine-grained control over extension loading.<commit_after># copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
# contact http://www.logilab.fr/ -- mailto:<EMAIL>
#
# This file is part of astroid.
#
# astroid is free software: you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by the
# Free Software Foundation, either version 2.1 of the License, or (at your
# option) any later version.
#
# astroid is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License
# for more details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with astroid. If not, see <http://www.gnu.org/licenses/>.
"""astroid manager: avoid multiple astroid build of a same module when
possible by providing a class responsible to get astroid representation
from various source and using a cache of built modules)
"""
from __future__ import print_function

__docformat__ = "restructuredtext en"

import collections
import imp
import os
from os.path import dirname, join, isdir, exists
from warnings import warn
import zipimport

from logilab.common.configuration import OptionsProviderMixIn

from astroid.exceptions import AstroidBuildingException
from astroid import modutils


def astroid_wrapper(func, modname):
    """wrapper to give to AstroidManager.project_from_files"""
    print('parsing %s...' % modname)
    try:
        return func(modname)
    except AstroidBuildingException as exc:
        print(exc)
    except Exception as exc:
        import traceback
        traceback.print_exc()

def _silent_no_wrap(func, modname):
    """silent wrapper that doesn't do anything; can be used for tests"""
    return func(modname)

def safe_repr(obj):
    try:
        return repr(obj)
    except:
        return '???'



class AstroidManager(OptionsProviderMixIn):
    """the astroid manager, responsible to build astroid from files
     or modules.

    Use the Borg pattern.
    """

    name = 'astroid loader'
    options = (("ignore",
                {'type' : "csv", 'metavar' : "<file>",
                 'dest' : "black_list", "default" : ('CVS',),
                 'help' : "add <file> (may be a directory) to the black list\
. It should be a base name, not a path. You may set this option multiple times\
."}),
               ("project",
                {'default': "No Name", 'type' : 'string', 'short': 'p',
                 'metavar' : '<project name>',
                 'help' : 'set the project name.'}),
              )
    brain = {}
    def __init__(self):
        self.__dict__ = AstroidManager.brain
        if not self.__dict__:
            OptionsProviderMixIn.__init__(self)
            self.load_defaults()
            # NOTE: cache entries are added by the [re]builder
            self.astroid_cache = {}
            self._mod_file_cache = {}
            self.transforms = collections.defaultdict(list)
            self._failed_import_hooks = []
            self.always_load_extensions = False
            self.extension_package_whitelist = set()

    def ast_from_file(self, filepath, modname=None, fallback=True, source=False):
        """given a module name, return the astroid object"""
        try:
            filepath = modutils.get_source_file(filepath, include_no_ext=True)
            source = True
        except modutils.NoSourceFile:
            pass
        if modname is None:
            try:
                modname = '.'.join(modutils.modpath_from_file(filepath))
            except ImportError:
                modname = filepath
        if modname in self.astroid_cache and self.astroid_cache[modname].file == filepath:
            return self.astroid_cache[modname]
        if source:
            from astroid.builder import AstroidBuilder
            return AstroidBuilder(self).file_build(filepath, modname)
        elif fallback and modname:
            return self.ast_from_module_name(modname)
        raise AstroidBuildingException('unable to get astroid for file %s' %
                                       filepath)

    def _build_stub_module(self, modname):
        from astroid.builder import AstroidBuilder
        return AstroidBuilder(self).string_build('', modname)

    def _can_load_extension(self, modname):
        if self.always_load_extensions:
            return True
        if modutils.is_standard_module(modname):
            return True
        parts = modname.split('.')
        return any(
            '.'.join(parts[:x]) in self.extension_package_whitelist
            for x in range(1, len(parts) + 1))

    def ast_from_module_name(self, modname, context_file=None):
        """given a module name, return the astroid object"""
        if modname in self.astroid_cache:
            return self.astroid_cache[modname]
        if modname == '__main__':
            return self._build_stub_module(modname)
        old_cwd = os.getcwd()
        if context_file:
            os.chdir(dirname(context_file))
        try:
            filepath, mp_type = self.file_from_module_name(modname, context_file)
            if mp_type == modutils.PY_ZIPMODULE:
                module = self.zip_import_data(filepath)
                if module is not None:
                    return module
            elif mp_type in (imp.C_BUILTIN, imp.C_EXTENSION):
                if mp_type == imp.C_EXTENSION and not self._can_load_extension(modname): 
                    return self._build_stub_module(modname)
                try:
                    module = modutils.load_module_from_name(modname)
                except Exception as ex:
                    msg = 'Unable to load module %s (%s)' % (modname, ex)
                    raise AstroidBuildingException(msg)
                return self.ast_from_module(module, modname)
            elif mp_type == imp.PY_COMPILED:
                raise AstroidBuildingException("Unable to load compiled module %s" % (modname,))
            if filepath is None:
                raise AstroidBuildingException("Unable to load module %s" % (modname,))
            return self.ast_from_file(filepath, modname, fallback=False)
        except AstroidBuildingException as e:
            for hook in self._failed_import_hooks:
                try:
                    return hook(modname)
                except AstroidBuildingException:
                    pass
            raise e
        finally:
            os.chdir(old_cwd)

    def zip_import_data(self, filepath):
        if zipimport is None:
            return None
        from astroid.builder import AstroidBuilder
{'fraction_non_alphanumeric': 0.08132274724427657, 'fraction_numerical': 0.0016958298003545826, 'mean_word_length': 2.3266666666666667, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '186904', 'n_tokens_mistral': 3576, 'n_tokens_neox': 3475, 'n_words': 1125}