<commit_before>#!/usr/bin/env python

""" Annex provides assistance with developing plugin-based tools.

With Annex you can load and reload plugins from various python modules without
the requirement that they exist on the PYTHONPATH.

"""

from __future__ import with_statement

import glob
import logging
import hashlib
import imp
import inspect
import os

from .version import __version__

logger = logging.getLogger("annex")

__author__ = "<NAME> <<EMAIL>>"


def _md5sum(file_path):
    """ Helper function that builds and md5sum from a file in chunks.

    Args:
        file_path: The path to the file you want an md5sum for.

    Returns:
        A string containing an md5sum.

    """
    md5 = hashlib.md5()

    with open(file_path) as md5_file:
        while True:
            data = md5_file.read(1024 * 1024 * 4)
            if not data:
                break
            md5.update(data)

    return md5.digest()


class PluginModule(object):
    """ Container for the plugins contained in a single module file.

    Attributes:
        plugin_file: Path to the file in which this plugin module was loaded.
        plugins: A list containing instances of plugins loaded from the plugin_file.
        md5: An md5 of the plugin_file's contents.

    """
    def __init__(self, plugin_file, plugins):
        self.plugin_file = plugin_file
        self.plugins = plugins
        self.md5 = _md5sum(plugin_file)


class Annex(object):
    """ Class for loading and retreiving various plugins.

    Attributes:
        base_plugin: Class from which your plugins subclass. This is used to
            decide which plugins will be loaded.
        plugin_dirs: A set of all directories where plugins can be loaded from.
        loaded_modules: A dictionary mapping plugin module filenames to
            PluginModule objects.

    """
    def __init__(self, base_plugin, plugin_dirs, instantiate=True):
        """ Initializes plugins given paths to directories containing plugins.

        Args:
            plugin_dirs: Can be a multilayered list of strings, which will be
                flattened, containing an absolute path to a directory
                containing plugins.
            instantiate: By default Annex will instantiate your plugins. Some
                times you need the control of instantiating them yourself.
        """

        self.base_plugin = base_plugin
        self.plugin_dirs = set()
        self.loaded_modules = {}
        self._instantiate = instantiate

        for plugin_dir in plugin_dirs:
            if isinstance(plugin_dir, basestring):
                self.plugin_dirs.add(plugin_dir)
            else:
                self.plugin_dirs.update(plugin_dir)

        for plugin_file in self._get_plugin_files(self.plugin_dirs):
            self._load_plugin(plugin_file)

    def __len__(self):
        return len(self.loaded_modules)

    def __iter__(self):
        for modules in self.loaded_modules.itervalues():
            for plugin in modules.plugins:
                yield plugin

    def __getattr__(self, name):
        for plugin in self:
            if plugin.__class__.__name__ == name:<commit_msg>bug fix with accessing by name when not instantiated.<commit_after>#!/usr/bin/env python

""" Annex provides assistance with developing plugin-based tools.

With Annex you can load and reload plugins from various python modules without
the requirement that they exist on the PYTHONPATH.

"""

from __future__ import with_statement

import glob
import logging
import hashlib
import imp
import inspect
import os

from .version import __version__

logger = logging.getLogger("annex")

__author__ = "<NAME> <<EMAIL>>"


def _md5sum(file_path):
    """ Helper function that builds and md5sum from a file in chunks.

    Args:
        file_path: The path to the file you want an md5sum for.

    Returns:
        A string containing an md5sum.

    """
    md5 = hashlib.md5()

    with open(file_path) as md5_file:
        while True:
            data = md5_file.read(1024 * 1024 * 4)
            if not data:
                break
            md5.update(data)

    return md5.digest()


class PluginModule(object):
    """ Container for the plugins contained in a single module file.

    Attributes:
        plugin_file: Path to the file in which this plugin module was loaded.
        plugins: A list containing instances of plugins loaded from the plugin_file.
        md5: An md5 of the plugin_file's contents.

    """
    def __init__(self, plugin_file, plugins):
        self.plugin_file = plugin_file
        self.plugins = plugins
        self.md5 = _md5sum(plugin_file)


class Annex(object):
    """ Class for loading and retreiving various plugins.

    Attributes:
        base_plugin: Class from which your plugins subclass. This is used to
            decide which plugins will be loaded.
        plugin_dirs: A set of all directories where plugins can be loaded from.
        loaded_modules: A dictionary mapping plugin module filenames to
            PluginModule objects.

    """
    def __init__(self, base_plugin, plugin_dirs, instantiate=True):
        """ Initializes plugins given paths to directories containing plugins.

        Args:
            plugin_dirs: Can be a multilayered list of strings, which will be
                flattened, containing an absolute path to a directory
                containing plugins.
            instantiate: By default Annex will instantiate your plugins. Some
                times you need the control of instantiating them yourself.
        """

        self.base_plugin = base_plugin
        self.plugin_dirs = set()
        self.loaded_modules = {}
        self._instantiate = instantiate

        for plugin_dir in plugin_dirs:
            if isinstance(plugin_dir, basestring):
                self.plugin_dirs.add(plugin_dir)
            else:
                self.plugin_dirs.update(plugin_dir)

        for plugin_file in self._get_plugin_files(self.plugin_dirs):
            self._load_plugin(plugin_file)

    def __len__(self):
        return len(self.loaded_modules)

    def __iter__(self):
        for modules in self.loaded_modules.itervalues():
            for plugin in modules.plugins:
                yield plugin

    def __getattr__(self, name):
        for plugin in self:
            if self._instantiate:
                obj_name = plugin.__class__.__name__
            else:
                obj_name = plugin.__name__
            if obj_name == name:
{'fraction_non_alphanumeric': 0.06262501923372826, 'fraction_numerical': 0.007078012001846438, 'mean_word_length': 2.3557046979865772, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2593223', 'n_tokens_mistral': 1796, 'n_tokens_neox': 1680, 'n_words': 630}