<commit_before>from __future__ import print_function

import argparse
import os
import pkgutil
import sys

from copy import deepcopy
from json import dumps
from signal import signal
from signal import SIGTERM, SIGUSR1
from subprocess import Popen
from subprocess import call
from threading import Event
from time import sleep, time
from syslog import syslog, LOG_ERR, LOG_INFO, LOG_WARNING

import py3status.docstrings as docstrings
from py3status import modules as sitepkg_modules
from py3status.events import Events
from py3status.helpers import print_line, print_stderr
from py3status.i3status import I3status
from py3status.module import Module
from py3status.profiling import profile


class Py3statusWrapper():
    """
    This is the py3status wrapper.
    """

    def __init__(self):
        """
        Useful variables we'll need.
        """
        self.last_refresh_ts = time()
        self.lock = Event()
        self.modules = {}
        self.py3_modules = []

    def get_config(self):
        """
        Create the py3status based on command line options we received.
        """
        # get home path
        home_path = os.path.expanduser('~')

        # defaults
        config = {
            'cache_timeout': 60,
            'include_paths': ['{}/.i3/py3status/'.format(home_path)],
            'interval': 1
        }

        # package version
        try:
            import pkg_resources
            version = pkg_resources.get_distribution('py3status').version
        except:
            version = 'unknown'
        config['version'] = version

        # i3status config file default detection
        # respect i3status' file detection order wrt issue #43
        i3status_config_file_candidates = [
            '{}/.i3status.conf'.format(home_path),
            '{}/.config/i3status/config'.format(os.environ.get(
                'XDG_CONFIG_HOME', home_path)), '/etc/i3status.conf',
            '{}/i3status/config'.format(os.environ.get('XDG_CONFIG_DIRS',
                                                       '/etc/xdg'))
        ]
        for fn in i3status_config_file_candidates:
            if os.path.isfile(fn):
                i3status_config_file_default = fn
                break
        else:
            # if none of the default files exists, we will default
            # to ~/.i3/i3status.conf
            i3status_config_file_default = '{}/.i3/i3status.conf'.format(
                home_path)

        # command line options
        parser = argparse.ArgumentParser(
            description='The agile, python-powered, i3status wrapper')
        parser = argparse.ArgumentParser(add_help=True)
        parser.add_argument('-c',
                            '--config',
                            action="store",
                            dest="i3status_conf",
                            type=str,
                            default=i3status_config_file_default,
                            help="path to i3status config file")
        parser.add_argument('-d',
                            '--debug',
                            action="store_true",
                            help="be verbose in syslog")
        parser.add_argument('-i',
                            '--include',
                            action="append",
                            dest="include_paths",
                            help="""include user-written modules from those
                            directories (default ~/.i3/py3status)""")
        parser.add_argument('-n',
                            '--interval',
                            action="store",
                            dest="interval",
                            type=float,
                            default=config['interval'],
                            help="update interval in seconds (default 1 sec)")
        parser.add_argument('-s',
                            '--standalone',
                            action="store_true",
                            help="standalone mode, do not use i3status")
        parser.add_argument('-t',
                            '--timeout',
                            action="store",
                            dest="cache_timeout",
                            type=int,
                            default=config['cache_timeout'],
                            help="""default injection cache timeout in seconds
                            (default 60 sec)""")
        parser.add_argument('-v',
                            '--version',
                            action="store_true",
                            help="""show py3status version and exit""")
        parser.add_argument('cli_command', nargs='*', help=argparse.SUPPRESS)

        options = parser.parse_args()

        if options.cli_command:
            config['cli_command'] = options.cli_command

        # only asked for version
        if options.version:
            from platform import python_version
            print('py3status version {} (python {})'.format(config['version'],
                                                            python_version()))
            sys.exit(0)

        # override configuration and helper variables
        config['cache_timeout'] = options.cache_timeout
        config['debug'] = options.debug
        if options.include_paths:
            config['include_paths'] = options.include_paths
        config['interval'] = int(options.interval)
        config['standalone'] = options.standalone
        config['i3status_config_path'] = options.i3status_conf

        # all done
        return config

    def get_user_modules(self):
        """
        Search configured include directories for user provided modules.

        user_modules: {
            'weather_yahoo': ('~/i3/py3status/', 'weather_yahoo.py')
        }
        """
        user_modules = {}
        for include_path in sorted(self.config['include_paths']):
            include_path = os.path.abspath(include_path) + '/'
            if not os.path.isdir(include_path):
                continue
            for f_name in sorted(os.listdir(include_path)):
                if not f_name.endswith('.py'):
                    continue
                module_name = f_name[:-3]
                user_modules[module_name] = (include_path, f_name)
        return user_modules

    def get_all_modules(self):
        """
        Search and yield all available py3status modules:
            - in the current python's implementation site-packages
            - provided by the user using the inclusion directories

        User provided modules take precedence over py3status generic modules.
        """
        all_modules = {}
        for importer, module_name, ispkg in \
                pkgutil.iter_modules(sitepkg_modules.__path__):
            if not ispkg:
                mod = importer.find_module(module_name)
                all_modules[module_name] = (mod, None)
        user_modules = self.get_user_modules()
        all_modules.update(user_modules)
        for module_name, module_info in sorted(all_modules.items()):
            yield (module_name, module_info)
<commit_msg>remove unused core.get_all_modules()<commit_after>from __future__ import print_function

import argparse
import os
import sys

from copy import deepcopy
from json import dumps
from signal import signal
from signal import SIGTERM, SIGUSR1
from subprocess import Popen
from subprocess import call
from threading import Event
from time import sleep, time
from syslog import syslog, LOG_ERR, LOG_INFO, LOG_WARNING

import py3status.docstrings as docstrings
from py3status.events import Events
from py3status.helpers import print_line, print_stderr
from py3status.i3status import I3status
from py3status.module import Module
from py3status.profiling import profile


class Py3statusWrapper():
    """
    This is the py3status wrapper.
    """

    def __init__(self):
        """
        Useful variables we'll need.
        """
        self.last_refresh_ts = time()
        self.lock = Event()
        self.modules = {}
        self.py3_modules = []

    def get_config(self):
        """
        Create the py3status based on command line options we received.
        """
        # get home path
        home_path = os.path.expanduser('~')

        # defaults
        config = {
            'cache_timeout': 60,
            'include_paths': ['{}/.i3/py3status/'.format(home_path)],
            'interval': 1
        }

        # package version
        try:
            import pkg_resources
            version = pkg_resources.get_distribution('py3status').version
        except:
            version = 'unknown'
        config['version'] = version

        # i3status config file default detection
        # respect i3status' file detection order wrt issue #43
        i3status_config_file_candidates = [
            '{}/.i3status.conf'.format(home_path),
            '{}/.config/i3status/config'.format(os.environ.get(
                'XDG_CONFIG_HOME', home_path)), '/etc/i3status.conf',
            '{}/i3status/config'.format(os.environ.get('XDG_CONFIG_DIRS',
                                                       '/etc/xdg'))
        ]
        for fn in i3status_config_file_candidates:
            if os.path.isfile(fn):
                i3status_config_file_default = fn
                break
        else:
            # if none of the default files exists, we will default
            # to ~/.i3/i3status.conf
            i3status_config_file_default = '{}/.i3/i3status.conf'.format(
                home_path)

        # command line options
        parser = argparse.ArgumentParser(
            description='The agile, python-powered, i3status wrapper')
        parser = argparse.ArgumentParser(add_help=True)
        parser.add_argument('-c',
                            '--config',
                            action="store",
                            dest="i3status_conf",
                            type=str,
                            default=i3status_config_file_default,
                            help="path to i3status config file")
        parser.add_argument('-d',
                            '--debug',
                            action="store_true",
                            help="be verbose in syslog")
        parser.add_argument('-i',
                            '--include',
                            action="append",
                            dest="include_paths",
                            help="""include user-written modules from those
                            directories (default ~/.i3/py3status)""")
        parser.add_argument('-n',
                            '--interval',
                            action="store",
                            dest="interval",
                            type=float,
                            default=config['interval'],
                            help="update interval in seconds (default 1 sec)")
        parser.add_argument('-s',
                            '--standalone',
                            action="store_true",
                            help="standalone mode, do not use i3status")
        parser.add_argument('-t',
                            '--timeout',
                            action="store",
                            dest="cache_timeout",
                            type=int,
                            default=config['cache_timeout'],
                            help="""default injection cache timeout in seconds
                            (default 60 sec)""")
        parser.add_argument('-v',
                            '--version',
                            action="store_true",
                            help="""show py3status version and exit""")
        parser.add_argument('cli_command', nargs='*', help=argparse.SUPPRESS)

        options = parser.parse_args()

        if options.cli_command:
            config['cli_command'] = options.cli_command

        # only asked for version
        if options.version:
            from platform import python_version
            print('py3status version {} (python {})'.format(config['version'],
                                                            python_version()))
            sys.exit(0)

        # override configuration and helper variables
        config['cache_timeout'] = options.cache_timeout
        config['debug'] = options.debug
        if options.include_paths:
            config['include_paths'] = options.include_paths
        config['interval'] = int(options.interval)
        config['standalone'] = options.standalone
        config['i3status_config_path'] = options.i3status_conf

        # all done
        return config

    def get_user_modules(self):
        """
        Search configured include directories for user provided modules.

        user_modules: {
            'weather_yahoo': ('~/i3/py3status/', 'weather_yahoo.py')
        }
        """
        user_modules = {}
        for include_path in sorted(self.config['include_paths']):
            include_path = os.path.abspath(include_path) + '/'
            if not os.path.isdir(include_path):
                continue
            for f_name in sorted(os.listdir(include_path)):
                if not f_name.endswith('.py'):
                    continue
                module_name = f_name[:-3]
                user_modules[module_name] = (include_path, f_name)
        return user_modules

{'fraction_non_alphanumeric': 0.0920305023923445, 'fraction_numerical': 0.008148923444976076, 'mean_word_length': 1.5504289799809343, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7274376', 'n_tokens_mistral': 3372, 'n_tokens_neox': 3281, 'n_words': 774}