<commit_before># -*- coding: utf-8 -*-

# Copyright(C) 2010  <NAME>, <NAME>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.


from __future__ import with_statement

import atexit
from cmd import Cmd
import getpass
import logging
from optparse import OptionGroup, OptionParser
import os
import re
import sys

from weboob.capabilities.base import FieldNotFound
from weboob.core import CallErrors
from weboob.core.backendscfg import BackendsConfig, BackendAlreadyExists
from weboob.tools.misc import iter_fields
from weboob.tools.browser import BrowserUnavailable, BrowserIncorrectPassword

from .base import BackendNotFound, BaseApplication
from .formatters.load import formatters as available_formatters, load_formatter
from .results import ResultsCondition, ResultsConditionException


__all__ = ['ReplApplication', 'NotEnoughArguments']


class NotEnoughArguments(Exception):
    pass

class ReplApplication(Cmd, BaseApplication):
    """
    Base application class for CLI applications.
    """

    SYNOPSIS =  'Usage: %prog [-dqv] [-b backends] [-cnfs] [command [arguments..]]\n'
    SYNOPSIS += '       %prog [--help] [--version]'
    CAPS = None

    # shell escape strings
    BOLD   = '[37;1m'
    NC     = '[37;0m'    # no color

    def __init__(self):
        Cmd.__init__(self)
        self.prompt = self.BOLD + '%s> ' % self.APPNAME + self.NC
        self.intro = '\n'.join(('Welcome to %s%s%s v%s' % (self.BOLD, self.APPNAME, self.NC, self.VERSION),
                                '',
                                '%s' % self.COPYRIGHT,
                                'This program is free software; you can redistribute it and/or modify',
                                'it under the terms of the GNU General Public License as published by',
                                'the Free Software Foundation, version 3 of the License.',
                                '',
                                'Type "help" to display available commands.',
                                '',
                               ))
        self.weboob_commands = ['backends', 'condition', 'count', 'formatter', 'logging', 'select', 'quit']
        self.hidden_commands = set(['EOF'])

        option_parser = OptionParser(self.SYNOPSIS, version=self._get_optparse_version())

        try:
            BaseApplication.__init__(self, option_parser=option_parser)
        except BackendsConfig.WrongPermissions, e:
            print e
            sys.exit(1)

        self._parser.format_description = lambda x: self._parser.description

        if self._parser.description is None:
            self._parser.description = ''

        app_cmds, weboob_cmds, undoc_cmds = self.get_commands_doc()
        help_str = '%s Commands:\n%s\n\n' % (self.APPNAME.capitalize(), '\n'.join(' %s' % cmd for cmd in sorted(app_cmds + undoc_cmds)))
        help_str +='Weboob Commands:\n%s\n' % '\n'.join(' %s' % cmd for cmd in weboob_cmds)
        self._parser.description += help_str

        results_options = OptionGroup(self._parser, 'Results Options')
        results_options.add_option('-c', '--condition', help='filter result items to display given a boolean expression')
        results_options.add_option('-n', '--count', default='10', type='int',
                                   help='get a maximum number of results (all backends merged)')
        results_options.add_option('-s', '--select', help='select result item keys to display (comma separated)')
        self._parser.add_option_group(results_options)

        formatting_options = OptionGroup(self._parser, 'Formatting Options')
        formatting_options.add_option('-f', '--formatter', choices=available_formatters,
                                      help='select output formatter (%s)' % u','.join(available_formatters))
        formatting_options.add_option('--no-header', dest='no_header', action='store_true', help='do not display header')
        formatting_options.add_option('--no-keys', dest='no_keys', action='store_true', help='do not display item keys')
        self._parser.add_option_group(formatting_options)

        try:
            import readline
        except ImportError:
            pass
        else:
            history_filepath = os.path.join(self.weboob.WORKDIR, '%s_history' % self.APPNAME)
            try:
                readline.read_history_file(history_filepath)
            except IOError:
                pass
            def savehist():
                readline.write_history_file(history_filepath)
            atexit.register(savehist)

        self._interactive = False
        self.enabled_backends = set()

    @property
    def interactive(self):
        return self._interactive

    def caps_included(self, modcaps, caps):
        modcaps = [x.__name__ for x in modcaps]
        if not isinstance(caps, (list,set,tuple)):
            caps = (caps,)
        for cap in caps:
            if not cap in modcaps:
                return False
        return True

    def edit_backend(self, name, params=None):
        return self.add_backend(name, params, True)

    def add_backend(self, name, params=None, edit=False):
        if params is None:
            params = {}

        if not edit:
            backend = self.weboob.modules_loader.get_or_load_module(name)
        else:
            bname, items = self.weboob.backends_config.get_backend(name)
            backend = self.weboob.modules_loader.get_or_load_module(bname)
            items.update(params)
            params = items
        if not backend:
            print 'Backend "%s" does not exist.' % name
            return None

        # ask for params non-specified on command-line arguments
        asked_config = False
        for key, value in backend.config.iteritems():
            if not asked_config:
                asked_config = True
                print 'Configuration of backend'
                print '------------------------'
            if key not in params or edit:
                params[key] = self.ask(' [%s] %s' % (key, value.description),
                                       default=params[key] if (edit and key in params) else value.default,
                                       masked=value.is_masked,
                                       choices=value.choices,
                                       regexp=value.regexp)
            else:
                print ' [%s] %s: %s' % (key, value.description, '(masked)' if value.is_masked else params[key])
        if asked_config:
            print '------------------------'

        try:
            self.weboob.backends_config.add_backend(name, name, params, edit=edit)
            print 'Backend "%s" successfully %s.' % (name, 'updated' if edit else 'added')
            return name
        except BackendAlreadyExists:
            print 'Backend "%s" is already configured in file "%s"' % (name, self.weboob.backends_config.confpath)
            while self.ask('Add new instance of "%s" backend?' % name, default=False):
                new_name = self.ask('Please give new instance name (could be "%s_1")' % name, regexp=u'^[\d\w_-]+$')
                try:
                    self.weboob.backends_config.add_backend(new_name, name, params)
                    print 'Backend "%s" successfully added.' % new_name
                    return new_name
                except BackendAlreadyExists:
                    print 'Instance "%s" already exists for backend "%s".' % (new_name, name)

    def unload_backends(self, *args, **kwargs):
        unloaded = self.weboob.unload_backends(*args, **kwargs)
        for backend in unloaded.itervalues():
            try:
                self.enabled_backends.remove(backend)
            except KeyError:
                pass
        return unloaded

    def load_backends(self, *args, **kwargs):
        ret = super(ReplApplication, self).load_backends(*args, **kwargs)
        for name, backend in ret.iteritems():
            self.enabled_backends.add(backend)
        while len(self.enabled_backends) == 0:<commit_msg>handle errors during loading and ask users to reconfigure backend<commit_after># -*- coding: utf-8 -*-

# Copyright(C) 2010  <NAME>, <NAME>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.


from __future__ import with_statement

import atexit
from cmd import Cmd
import getpass
import logging
from optparse import OptionGroup, OptionParser
import os
import re
import sys

from weboob.capabilities.base import FieldNotFound
from weboob.core import CallErrors
from weboob.core.backendscfg import BackendsConfig, BackendAlreadyExists
from weboob.tools.misc import iter_fields
from weboob.tools.browser import BrowserUnavailable, BrowserIncorrectPassword

from .base import BackendNotFound, BaseApplication
from .formatters.load import formatters as available_formatters, load_formatter
from .results import ResultsCondition, ResultsConditionException


__all__ = ['ReplApplication', 'NotEnoughArguments']


class NotEnoughArguments(Exception):
    pass

class ReplApplication(Cmd, BaseApplication):
    """
    Base application class for CLI applications.
    """

    SYNOPSIS =  'Usage: %prog [-dqv] [-b backends] [-cnfs] [command [arguments..]]\n'
    SYNOPSIS += '       %prog [--help] [--version]'
    CAPS = None

    # shell escape strings
    BOLD   = '[37;1m'
    NC     = '[37;0m'    # no color

    def __init__(self):
        Cmd.__init__(self)
        self.prompt = self.BOLD + '%s> ' % self.APPNAME + self.NC
        self.intro = '\n'.join(('Welcome to %s%s%s v%s' % (self.BOLD, self.APPNAME, self.NC, self.VERSION),
                                '',
                                '%s' % self.COPYRIGHT,
                                'This program is free software; you can redistribute it and/or modify',
                                'it under the terms of the GNU General Public License as published by',
                                'the Free Software Foundation, version 3 of the License.',
                                '',
                                'Type "help" to display available commands.',
                                '',
                               ))
        self.weboob_commands = ['backends', 'condition', 'count', 'formatter', 'logging', 'select', 'quit']
        self.hidden_commands = set(['EOF'])

        option_parser = OptionParser(self.SYNOPSIS, version=self._get_optparse_version())

        try:
            BaseApplication.__init__(self, option_parser=option_parser)
        except BackendsConfig.WrongPermissions, e:
            print e
            sys.exit(1)

        self._parser.format_description = lambda x: self._parser.description

        if self._parser.description is None:
            self._parser.description = ''

        app_cmds, weboob_cmds, undoc_cmds = self.get_commands_doc()
        help_str = '%s Commands:\n%s\n\n' % (self.APPNAME.capitalize(), '\n'.join(' %s' % cmd for cmd in sorted(app_cmds + undoc_cmds)))
        help_str +='Weboob Commands:\n%s\n' % '\n'.join(' %s' % cmd for cmd in weboob_cmds)
        self._parser.description += help_str

        results_options = OptionGroup(self._parser, 'Results Options')
        results_options.add_option('-c', '--condition', help='filter result items to display given a boolean expression')
        results_options.add_option('-n', '--count', default='10', type='int',
                                   help='get a maximum number of results (all backends merged)')
        results_options.add_option('-s', '--select', help='select result item keys to display (comma separated)')
        self._parser.add_option_group(results_options)

        formatting_options = OptionGroup(self._parser, 'Formatting Options')
        formatting_options.add_option('-f', '--formatter', choices=available_formatters,
                                      help='select output formatter (%s)' % u','.join(available_formatters))
        formatting_options.add_option('--no-header', dest='no_header', action='store_true', help='do not display header')
        formatting_options.add_option('--no-keys', dest='no_keys', action='store_true', help='do not display item keys')
        self._parser.add_option_group(formatting_options)

        try:
            import readline
        except ImportError:
            pass
        else:
            history_filepath = os.path.join(self.weboob.WORKDIR, '%s_history' % self.APPNAME)
            try:
                readline.read_history_file(history_filepath)
            except IOError:
                pass
            def savehist():
                readline.write_history_file(history_filepath)
            atexit.register(savehist)

        self._interactive = False
        self.enabled_backends = set()

    @property
    def interactive(self):
        return self._interactive

    def caps_included(self, modcaps, caps):
        modcaps = [x.__name__ for x in modcaps]
        if not isinstance(caps, (list,set,tuple)):
            caps = (caps,)
        for cap in caps:
            if not cap in modcaps:
                return False
        return True

    def edit_backend(self, name, params=None):
        return self.add_backend(name, params, True)

    def add_backend(self, name, params=None, edit=False):
        if params is None:
            params = {}

        if not edit:
            backend = self.weboob.modules_loader.get_or_load_module(name)
        else:
            bname, items = self.weboob.backends_config.get_backend(name)
            backend = self.weboob.modules_loader.get_or_load_module(bname)
            items.update(params)
            params = items
        if not backend:
            print 'Backend "%s" does not exist.' % name
            return None

        # ask for params non-specified on command-line arguments
        asked_config = False
        for key, value in backend.config.iteritems():
            if not asked_config:
                asked_config = True
                print 'Configuration of backend'
                print '------------------------'
            if key not in params or edit:
                params[key] = self.ask(' [%s] %s' % (key, value.description),
                                       default=params[key] if (edit and key in params) else value.default,
                                       masked=value.is_masked,
                                       choices=value.choices,
                                       regexp=value.regexp)
            else:
                print ' [%s] %s: %s' % (key, value.description, '(masked)' if value.is_masked else params[key])
        if asked_config:
            print '------------------------'

        try:
            self.weboob.backends_config.add_backend(name, name, params, edit=edit)
            print 'Backend "%s" successfully %s.' % (name, 'updated' if edit else 'added')
            return name
        except BackendAlreadyExists:
            print 'Backend "%s" is already configured in file "%s"' % (name, self.weboob.backends_config.confpath)
            while self.ask('Add new instance of "%s" backend?' % name, default=False):
                new_name = self.ask('Please give new instance name (could be "%s_1")' % name, regexp=u'^[\d\w_-]+$')
                try:
                    self.weboob.backends_config.add_backend(new_name, name, params)
                    print 'Backend "%s" successfully added.' % new_name
                    return new_name
                except BackendAlreadyExists:
                    print 'Instance "%s" already exists for backend "%s".' % (new_name, name)

    def unload_backends(self, *args, **kwargs):
        unloaded = self.weboob.unload_backends(*args, **kwargs)
        for backend in unloaded.itervalues():
            try:
                self.enabled_backends.remove(backend)
            except KeyError:
                pass
        return unloaded

    def load_backends(self, *args, **kwargs):
        if 'errors' in kwargs:
            errors = kwargs['errors']
        else:
            kwargs['errors'] = errors = []
        ret = super(ReplApplication, self).load_backends(*args, **kwargs)

        for err in errors:
            print >>sys.stderr, 'Error(%s): %s' % (err.backend_name, err)
            if self.ask('Do you want to reconfigure this backend?', default=True):
                self.edit_backend(err.backend_name)
                self.load_backends(names=[err.backend_name])

        for name, backend in ret.iteritems():
            self.enabled_backends.add(backend)
        while len(self.enabled_backends) == 0:
{'fraction_non_alphanumeric': 0.10499013250634338, 'fraction_numerical': 0.003608683394417818, 'mean_word_length': 2.1536273115220483, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4905381', 'n_tokens_mistral': 4919, 'n_tokens_neox': 4724, 'n_words': 1384}