<commit_before>"""
User-interface related functions for building bots.

This module supports several different bot classes which could be used in
conjunction. Each bot should subclass at least one of these four classes:

* L{BaseBot}: Basic bot class in case where the site is handled differently,
  like working on two sites in parallel.

* L{SingleSiteBot}: Bot class which should only be run on a single site. They
  usually store site specific content and thus can't be easily run when the
  generator returns a page on another site. It has a property C{site} which
  can also be changed. If the generator returns a page of a different site
  it'll skip that page.

* L{MultipleSitesBot}: Bot class which supports to be run on multiple sites
  without the need to manually initialize it every time. It is not possible to
  set the C{site} property and it's deprecated to request it. Instead site of
  the current page should be used. And out of C{run} that sit isn't defined.

* L{ConfigParserBot}: Bot class which supports reading options from a
  scripts.ini configuration file. That file consists of sections, led by a
  C{[section]} header and followed by C{option: value} or C{option=value}
  entries. The section is the script name without .py suffix. All options
  identified must be predefined in available_options dictionary.

* L{Bot}: The previous base class which should be avoided. This class is mainly
  used for bots which work with wikibase or together with an image repository.

Additionally there is the L{CurrentPageBot} class which automatically sets the
current page to the page treated. It is recommended to use this class and to
use C{treat_page} instead of C{treat} and C{put_current} instead of C{userPut}.
It by default subclasses the C{BaseBot} class.

With L{CurrentPageBot} it's possible to subclass one of the following classes
to filter the pages which are ultimately handled by C{treat_page}:

* L{ExistingPageBot}: Only handle pages which do exist.
* L{CreatingPageBot}: Only handle pages which do not exist.
* L{RedirectPageBot}: Only handle pages which are redirect pages.
* L{NoRedirectPageBot}: Only handle pages which are not redirect pages.
* L{FollowRedirectPageBot}: If the generator returns a redirect page it'll
  follow the redirect and instead work on the redirected class.

It is possible to combine filters by subclassing multiple of them. They are
new-style classes so when a class is first subclassing L{ExistingPageBot} and
then L{FollowRedirectPageBot} it will also work on pages which do not exist
when a redirect pointed to that. If the order is inversed it'll first follow
them and then check whether they exist.

Additionally there is the L{AutomaticTWSummaryBot} which subclasses
L{CurrentPageBot} and automatically defines the summary when C{put_current} is
used.
"""
#
# (C) Pywikibot team, 2008-2021
#
# Distributed under the terms of the MIT license.
#
__all__ = (
    'CRITICAL', 'ERROR', 'INFO', 'WARNING', 'DEBUG', 'INPUT', 'STDOUT',
    'VERBOSE', 'critical', 'debug', 'error', 'exception', 'log', 'warning',
    'output', 'stdout', 'LoggingFormatter', 'RotatingFileHandler',
    'init_handlers', 'writelogheader',
    'input', 'input_choice', 'input_yn', 'input_list_choice', 'ui',
    'Option', 'StandardOption', 'NestedOption', 'IntegerOption',
    'ContextOption', 'ListOption', 'ShowingListOption', 'MultipleChoiceList',
    'ShowingMultipleChoiceList', 'OutputProxyOption',
    'HighlightContextOption', 'ChoiceException', 'UnhandledAnswer',
    'Choice', 'StaticChoice', 'LinkChoice', 'AlwaysChoice',
    'QuitKeyboardInterrupt',
    'InteractiveReplace',
    'calledModuleName', 'handle_args',
    'show_help', 'showHelp', 'suggest_help',
    'writeToCommandLogFile', 'open_webbrowser',
    'OptionHandler',
    'BaseBot', 'Bot', 'ConfigParserBot', 'SingleSiteBot', 'MultipleSitesBot',
    'CurrentPageBot', 'AutomaticTWSummaryBot',
    'ExistingPageBot', 'FollowRedirectPageBot', 'CreatingPageBot',
    'RedirectPageBot', 'NoRedirectPageBot',
    'WikidataBot',
)


import codecs
import configparser
import datetime
import json
import logging
import logging.handlers
import os
import sys
import time
import warnings
import webbrowser

from contextlib import closing
from importlib import import_module
from pathlib import Path
from textwrap import fill
from typing import Any, Optional, Union
from warnings import warn

import pywikibot
from pywikibot.backports import Dict, Iterable, List, Sequence
from pywikibot import config2 as config
from pywikibot import daemonize
from pywikibot import i18n
from pywikibot import version
from pywikibot.bot_choice import (
    Option, StandardOption, NestedOption, IntegerOption, ContextOption,
    ListOption, ShowingListOption, MultipleChoiceList,
    ShowingMultipleChoiceList, OutputProxyOption, HighlightContextOption,
    ChoiceException, QuitKeyboardInterrupt,
    Choice, StaticChoice, LinkChoice, AlwaysChoice
)
from pywikibot.exceptions import UnknownFamily, UnknownSite
from pywikibot.logging import CRITICAL, ERROR, INFO, WARNING
from pywikibot.logging import DEBUG, INPUT, STDOUT, VERBOSE
from pywikibot.logging import (
    add_init_routine,
    critical, debug, error, exception, log, output, stdout, warning,
)
from pywikibot.tools import (
    deprecated, deprecate_arg, deprecated_args, issue_deprecation_warning,
    PYTHON_VERSION, suppress_warnings
)
from pywikibot.tools._logging import LoggingFormatter, RotatingFileHandler
from pywikibot.tools.formatter import color_format


# Note: all output goes through python std library "logging" module
_logger = 'bot'

# User interface initialization
# search for user interface module in the 'userinterfaces' subdirectory
uiModule = __import__('pywikibot.userinterfaces.%s_interface'
                      % config.userinterface,
                      fromlist=['UI'])
ui = uiModule.UI()
pywikibot.argvu = ui.argvu()

_GLOBAL_HELP = """
GLOBAL OPTIONS
==============
(Global arguments available for all bots)

-dir:PATH         Read the bot's configuration data from directory given by
                  PATH, instead of from the default directory.

-lang:xx          Set the language of the wiki you want to work on, overriding
                  the configuration in user-config.py. xx should be the
                  site code.

-family:xyz       Set the family of the wiki you want to work on, e.g.
                  wikipedia, wiktionary, wikivoyage, ...
                  This will override the configuration in user-config.py.

-site:xyz:xx      Set the wiki site you want to work on, e.g.
                  wikipedia:test, wiktionary:de, wikivoyage:en, ...
                  This will override the configuration in user-config.py.

-user:xyz         Log in as user 'xyz' instead of the default username.

-daemonize:xyz    Immediately return control to the terminal and redirect
                  stdout and stderr to file xyz.
                  (only use for bots that require no input from stdin).

-help             Show this help text.

-log              Enable the log file, using the default filename
                  '{}-bot.log'
                  Logs will be stored in the logs subdirectory.

-log:xyz          Enable the log file, using 'xyz' as the filename.

-nolog            Disable the log file (if it is enabled by default).

-maxlag           Sets a new maxlag parameter to a number of seconds. Defer bot
                  edits during periods of database server lag. Default is set
                  by config.py

-putthrottle:n    Set the minimum time (in seconds) the bot will wait between
-pt:n             saving pages.
-put_throttle:n

-debug:item       Enable the log file and include extensive debugging data
-debug            for component "item" (for all components if the second form
                  is used).

-verbose          Have the bot provide additional console output that may be
-v                useful in debugging.

-cosmeticchanges  Toggles the cosmetic_changes setting made in config.py or
-cc               user-config.py to its inverse and overrules it. All other
                  settings and restrictions are untouched.

-simulate         Disables writing to the server. Useful for testing and
                  debugging of new code (if given, doesn't do any real
                  changes, but only shows what would have been changed).
                  An integer or float value may be given to simulate a
                  processing time; the bot just waits for given seconds.

-<config var>:n   You may use all given numeric config variables as option and
                  modify it with command line.

"""

_GLOBAL_HELP_NOTE = """
GLOBAL OPTIONS
==============
For global options use -help:global or run pwb.py -help

"""


class UnhandledAnswer(Exception):

    """The given answer didn't suffice."""

    def __init__(self, stop=False):
        """Initializer."""
        self.stop = stop


# Initialize the handlers and formatters for the logging system.
#
# This relies on the global variable 'ui' which is a UserInterface object
# defined in the 'userinterface' subpackage.
#
# The UserInterface object must define its own init_handlers() method
# which takes the root logger as its only argument, and which adds to that
# logger whatever handlers and formatters are needed to process output and
# display it to the user. The default (terminal) interface sends level
# STDOUT to sys.stdout (as all interfaces should) and sends all other
# levels to sys.stderr; levels WARNING and above are labeled with the
# level name.
#
# UserInterface objects must also define methods input(), input_choice(),
# and editText(), all of which are documented in
# userinterfaces/terminal_interface.py

_handlers_initialized = False


def init_handlers(strm=None):
    """Initialize logging system for terminal-based bots.

    This function must be called before using pywikibot.output(); and must
    be called again if the destination stream is changed.

    Note: this function is called by handle_args(), so it should normally
    not need to be called explicitly

    All user output is routed through the logging module.
    Each type of output is handled by an appropriate handler object.
    This structure is used to permit eventual development of other
    user interfaces (GUIs) without modifying the core bot code.

    The following output levels are defined:

     - DEBUG: only for file logging; debugging messages.
     - STDOUT: output that must be sent to sys.stdout (for bots that may
       have their output redirected to a file or other destination).
     - VERBOSE: optional progress information for display to user.
     - INFO: normal (non-optional) progress information for display to user.
     - INPUT: prompts requiring user response.
     - WARN: user warning messages.
     - ERROR: user error messages.
     - CRITICAL: fatal error messages.

    Accordingly, do **not** use print statements in bot code; instead,
    use pywikibot.output function.

    @param strm: Output stream. If None, re-uses the last stream if one
        was defined, otherwise uses sys.stderr

    """
    global _handlers_initialized

    module_name = calledModuleName()
    if not module_name:
        module_name = 'terminal-interface'

    logging.addLevelName(VERBOSE, 'VERBOSE')
    # for messages to be displayed on terminal at "verbose" setting
    # use INFO for messages to be displayed even on non-verbose setting

    logging.addLevelName(STDOUT, 'STDOUT')
    # for messages to be displayed to stdout

    logging.addLevelName(INPUT, 'INPUT')
    # for prompts requiring user response

    root_logger = logging.getLogger('pywiki')
    root_logger.setLevel(DEBUG + 1)  # all records except DEBUG go to logger

    warnings_logger = logging.getLogger('py.warnings')
    warnings_logger.setLevel(DEBUG)

    # If there are command line warnings options, do not override them
    if not sys.warnoptions:
        logging.captureWarnings(True)

        if config.debug_log or 'deprecation' in config.log:
            warnings.filterwarnings('always')
        elif config.verbose_output:
            warnings.filterwarnings('module')
        warnings.filterwarnings('once', category=FutureWarning)

    root_logger.handlers = []  # remove any old handlers

    # configure handler(s) for display to user interface
    ui.init_handlers(root_logger, **config.userinterface_init_kwargs)

    # if user has enabled file logging, configure file handler
    if module_name in config.log or '*' in config.log:
        if config.logfilename:
            logfile = config.datafilepath('logs', config.logfilename)
        else:
            logfile = config.datafilepath('logs', '%s-bot.log' % module_name)
        file_handler = RotatingFileHandler(filename=logfile,
                                           maxBytes=1024 * config.logfilesize,
                                           backupCount=config.logfilecount,
                                           encoding='utf-8')

        file_handler.setLevel(DEBUG)
        form = LoggingFormatter(
            fmt='%(asctime)s %(caller_file)18s, %(caller_line)4s '
                'in %(caller_name)18s: %(levelname)-8s %(message)s',
            datefmt='%Y-%m-%d %H:%M:%S'
        )
        file_handler.setFormatter(form)
        root_logger.addHandler(file_handler)
        # Turn on debugging for each component requested by user
        # or for all components if nothing was specified
        for component in config.debug_log:
            if component:
                debuglogger = logging.getLogger('pywiki.' + component)
            else:
                debuglogger = logging.getLogger('pywiki')
            debuglogger.setLevel(DEBUG)
            debuglogger.addHandler(file_handler)

        warnings_logger.addHandler(file_handler)

    _handlers_initialized = True

    writelogheader()


def writelogheader():
    """
    Save additional version, system and status info to the log file in use.

    This may help the user to track errors or report bugs.
    """
    log('')
    log('=== Pywikibot framework v{} -- Logging header ==='
        .format(pywikibot.__version__))

    # script call
    log('COMMAND: {}'.format(sys.argv))

    # script call time stamp
    log('DATE: {} UTC'.format(datetime.datetime.utcnow()))

    # new framework release/revision? (handle_args needs to be called first)
    try:
        log('VERSION: {}'.format(
            version.getversion(online=config.log_pywiki_repo_version).strip()))
    except version.ParseError:
        exception()

    # system
    if hasattr(os, 'uname'):
        log('SYSTEM: {}'.format(os.uname()))

    # config file dir
    log('CONFIG FILE DIR: {}'.format(pywikibot.config2.base_dir))

    all_modules = sys.modules.keys()

    # These are the main dependencies of pywikibot.
    check_package_list = [
        'requests',
        'mwparserfromhell',
    ]

    # report all imported packages
    if config.verbose_output:
        check_package_list += all_modules

    log('PACKAGES:')
    packages = version.package_versions(check_package_list)
    for name in sorted(packages.keys()):
        info = packages[name]
        info.setdefault('path',
                        '[{}]'.format(info.get('type', 'path unknown')))
        info.setdefault('ver', '??')
        if 'err' in info:
            log('  {name}: {err}'.format_map(info))
        else:
            log('  {name} ({path}) = {ver}'.format_map(info))

    # imported modules
    log('MODULES:')
    for module in sys.modules.values():
        filename = version.get_module_filename(module)
        if not filename:
            continue

        param = {'sep': ' '}
        if PYTHON_VERSION >= (3, 6, 0):
            param['timespec'] = 'seconds'
        mtime = version.get_module_mtime(module).isoformat(**param)

        log('  {} {}'
            .format(mtime, filename))

    if config.log_pywiki_repo_version:
        log('PYWIKI REPO VERSION: {}'.format(version.getversion_onlinerepo()))

    log('=' * 57)


add_init_routine(init_handlers)


# User input functions


def input(question: str, password: bool = False,
          default: str = '', force: bool = False) -> str:
    """Ask the user a question, return the user's answer.

    @param question: a string that will be shown to the user. Don't add a
        space after the question mark/colon, this method will do this for you.
    @param password: if True, hides the user's input (for password entry).
    @param default: The default answer if none was entered. None to require
        an answer.
    @param force: Automatically use the default
    """
    # make sure logging system has been initialized
    if not _handlers_initialized:
        init_handlers()

    data = ui.input(question, password=password, default=default, force=force)
    return data


def input_choice(question: str, answers, default: Optional[str] = None,
                 return_shortcut: bool = True,
                 automatic_quit: bool = True, force: bool = False):
    """
    Ask the user the question and return one of the valid answers.

    @param question: The question asked without trailing spaces.
    @param answers: The valid answers each containing a full length answer and
        a shortcut. Each value must be unique.
    @type answers: iterable containing a sequence of length two or instances of
        ChoiceException
    @param default: The result if no answer was entered. It must not be in the
        valid answers and can be disabled by setting it to None. If it should
        be linked with the valid answers it must be its shortcut.
    @param return_shortcut: Whether the shortcut or the index of the answer is
        returned.
    @param automatic_quit: Adds the option 'Quit' ('q') and throw a
        L{QuitKeyboardInterrupt} if selected.
    @param force: Automatically use the default
    @return: The selected answer shortcut or index. Is -1 if the default is
        selected, it does not return the shortcut and the default is not a
        valid shortcut.
    @rtype: int (if not return shortcut), str (otherwise)
    """
    # make sure logging system has been initialized
    if not _handlers_initialized:
        init_handlers()

    return ui.input_choice(question, answers, default, return_shortcut,
                           automatic_quit=automatic_quit, force=force)


def input_yn(question: str,
             default: Union[str, bool, None] = None,
             automatic_quit: bool = True,
             force: bool = False) -> bool:
    """
    Ask the user a yes/no question and return the answer as a bool.

    @param question: The question asked without trailing spaces.
    @param default: The result if no answer was entered. It must be a bool or
        'y' or 'n' and can be disabled by setting it to None.
    @param automatic_quit: Adds the option 'Quit' ('q') and throw a
        L{QuitKeyboardInterrupt} if selected.
    @param force: Automatically use the default
    @return: Return True if the user selected yes and False if the user
        selected no. If the default is not None it'll return True if default
        is True or 'y' and False if default is False or 'n'.
    """
    if default not in ['y', 'Y', 'n', 'N']:
        if default:
            default = 'y'
        elif default is not None:
            default = 'n'
    assert default in ['y', 'Y', 'n', 'N', None], \
        'Default choice must be one of YyNn or default'

    return input_choice(question, [('Yes', 'y'), ('No', 'n')], default,
                        automatic_quit=automatic_quit, force=force) == 'y'


def input_list_choice(question: str, answers: Sequence[Any],
                      default: Union[int, str, None] = None,
                      force: bool = False) -> str:
    """
    Ask the user the question and return one of the valid answers.

    @param question: The question asked without trailing spaces.
    @param answers: The valid answers each containing a full length answer.
    @param default: The result if no answer was entered. It must not be in the
        valid answers and can be disabled by setting it to None.
    @param force: Automatically use the default
    @return: The selected answer.
    """
    if not _handlers_initialized:
        init_handlers()

    return ui.input_list_choice(question, answers, default=default,
                                force=force)


class InteractiveReplace:

    """
    A callback class for textlib's replace_links.

    It shows various options which can be switched on and off:
    * allow_skip_link = True (skip the current link)
    * allow_unlink = True (unlink)
    * allow_replace = False (just replace target, keep section and label)
    * allow_replace_section = False (replace target and section, keep label)
    * allow_replace_label = False (replace target and label, keep section)
    * allow_replace_all = False (replace target, section and label)
    (The boolean values are the default values)

    It has also a C{context} attribute which must be a non-negative integer. If
    it is greater 0 it shows that many characters before and after the link in
    question. The C{context_delta} attribute can be defined too and adds an
    option to increase C{context} by the given amount each time the option is
    selected.

    Additional choices can be defined using the 'additional_choices' and will
    be amended to the choices defined by this class. This list is mutable and
    the Choice instance returned and created by this class are too.
    """

    def __init__(self, old_link, new_link, default: Optional[str] = None,
                 automatic_quit: bool = True):
        """
        Initializer.

        @param old_link: The old link which is searched. The label and section
            are ignored.
        @type old_link: pywikibot.page.Link or pywikibot.page.Page
        @param new_link: The new link with which it should be replaced.
            Depending on the replacement mode it'll use this link's label and
            section. If False it'll unlink all and the attributes beginning
            with allow_replace are ignored.
        @type new_link: pywikibot.page.Link or pywikibot.page.Page or False
        @param default: The default answer as the shortcut
        @param automatic_quit: Add an option to quit and raise a
            QuitKeyboardException.
        """
        if isinstance(old_link, pywikibot.Page):
            self._old = old_link._link
        else:
            self._old = old_link
        if isinstance(new_link, pywikibot.Page):
            self._new = new_link._link
        else:
            self._new = new_link
        self._default = default
        self._quit = automatic_quit
        self._current_match = None
        self.context = 30
        self.context_delta = 0
        self.allow_skip_link = True
        self.allow_unlink = True
        self.allow_replace = False
        self.allow_replace_section = False
        self.allow_replace_label = False
        self.allow_replace_all = False
        # Use list to preserve order
        self._own_choices = [
            ('skip_link', StaticChoice('Do not change', 'n', None)),
            ('unlink', StaticChoice('Unlink', 'u', False)),
        ]
        if self._new:
            self._own_choices += [
                ('replace', LinkChoice('Change link target', 't', self,
                                       False, False)),
                ('replace_section', LinkChoice(
                    'Change link target and section', 's', self, True, False)),
                ('replace_label', LinkChoice('Change link target and label',
                                             'l', self, False, True)),
                ('replace_all', LinkChoice('Change complete link', 'c', self,
                                           True, True)),
            ]

        self.additional_choices = []

    def handle_answer(self, choice):
        """Return the result for replace_links."""
        for c in self.choices:
            if c.shortcut == choice:
                return c.handle()
        else:
            raise ValueError('Invalid choice "{0}"'.format(choice))

    def __call__(self, link, text, groups, rng):
        """Ask user how the selected link should be replaced."""
        if self._old == link:
            self._current_match = (link, text, groups, rng)
            while True:
                try:
                    answer = self.handle_link()
                except UnhandledAnswer as e:
                    if e.stop:
                        raise
                else:
                    break
            self._current_match = None  # don't reset in case of an exception
            return answer
        else:
            return None

    @property
    def choices(self):
        """Return the tuple of choices."""
        choices = []
        for name, choice in self._own_choices:
            if getattr(self, 'allow_' + name):
                choices += [choice]
        if self.context_delta > 0:
            choices += [HighlightContextOption(
                'more context', 'm', self.current_text, self.context,
                self.context_delta, *self.current_range)]
        choices += self.additional_choices
        return tuple(choices)

    def handle_link(self):
        """Handle the currently given replacement."""
        choices = self.choices
        for choice in choices:
            if isinstance(choice, Choice) and choice.handle_link():
                return choice.answer

        if self.context > 0:
            rng = self.current_range
            text = self.current_text
            # at the beginning of the link, start red color.
            # at the end of the link, reset the color to default
            pywikibot.output(text[max(0, rng[0] - self.context): rng[0]]
                             + color_format('{lightred}{0}{default}',
                                            text[rng[0]: rng[1]])
                             + text[rng[1]: rng[1] + self.context])
            question = 'Should the link '
        else:
            question = 'Should the link {lightred}{0}{default} '

        if self._new is False:
            question += 'be unlinked?'
        else:
            question += color_format('target to {lightpurple}{0}{default}?',
                                     self._new.canonical_title())

        choice = pywikibot.input_choice(
            color_format(question, self._old.canonical_title()),
            choices, default=self._default, automatic_quit=self._quit)

        return self.handle_answer(choice)

    @property
    def current_link(self):
        """Get the current link when it's handling one currently."""
        if self._current_match is None:
            raise ValueError('No current link')
        return self._current_match[0]

    @property
    def current_text(self):
        """Get the current text when it's handling one currently."""
        if self._current_match is None:
            raise ValueError('No current text')
        return self._current_match[1]

    @property
    def current_groups(self):
        """Get the current groups when it's handling one currently."""
        if self._current_match is None:
            raise ValueError('No current groups')
        return self._current_match[2]

    @property
    def current_range(self):
        """Get the current range when it's handling one currently."""
        if self._current_match is None:
            raise ValueError('No current range')
        return self._current_match[3]


# Command line parsing and help
def calledModuleName() -> str:
    """Return the name of the module calling this function.

    This is required because the -help option loads the module's docstring
    and because the module name will be used for the filename of the log.

    """
    return Path(pywikibot.argvu[0]).stem


def handle_args(args: Optional[Iterable[str]] = None,
                do_help: bool = True) -> List[str]:
    """
    Handle standard command line arguments, and return the rest as a list.

    Takes the command line arguments as strings, processes all global
    parameters such as -lang or -log, initialises the logging layer,
    which emits startup information into log at level 'verbose'.

    This makes sure that global arguments are applied first,
    regardless of the order in which the arguments were given.

    args may be passed as an argument, thereby overriding sys.argv

    @param args: Command line arguments
    @param do_help: Handle parameter '-help' to show help and invoke sys.exit
    @return: list of arguments not recognised globally
    """
    if pywikibot._sites:
        warn('Site objects have been created before arguments were handled',
             UserWarning)

    # get commandline arguments if necessary
    if not args:
        # it's the version in pywikibot.__init__ that is changed by scripts,
        # not the one in pywikibot.bot.
        args = pywikibot.argvu[1:]

    # get the name of the module calling this function. This is
    # required because the -help option loads the module's docstring and
    # because the module name will be used for the filename of the log.
    module_name = calledModuleName() or 'terminal-interface'
    non_global_args = []
    username = None
    do_help = None if do_help else False
    for arg in args:
        option, _, value = arg.partition(':')
        if do_help is not False and option == '-help':
            do_help = value or True
        elif option == '-dir':
            pass
        elif option == '-site':
            config.family, config.mylang = value.split(':')
        elif option == '-family':
            config.family = value
        elif option == '-lang':
            config.mylang = value
        elif option == '-user':
            username = value
        elif option in ('-putthrottle', '-pt'):
            config.put_throttle = float(value)
        elif option == '-log':
            if module_name not in config.log:
                config.log.append(module_name)
            if value:
                config.logfilename = value
        elif option == '-nolog':
            config.log = []
        elif option in ('-cosmeticchanges', '-cc'):
            config.cosmetic_changes = not config.cosmetic_changes
            output('NOTE: option cosmetic_changes is %s\n'
                   % config.cosmetic_changes)
        elif option == '-simulate':
            config.simulate = value or True
        #
        #  DEBUG control:
        #
        #    The framework has four layers (by default, others can be added),
        #    each designated by a string --
        #
        #    1.  "comm": the communication layer (http requests, etc.)
        #    2.  "data": the raw data layer (API requests, XML dump parsing)
        #    3.  "wiki": the wiki content representation layer (Page and Site
        #         objects)
        #    4.  "bot": the application layer (user scripts should always
        #         send any debug() messages to this layer)
        #
        #    The "-debug:layer" flag sets the logger for any specified
        #    layer to the DEBUG level, causing it to output extensive debugging
        #    information. Otherwise, the default logging setting is the INFO
        #    level. "-debug" with no layer specified sets _all_ loggers to
        #    DEBUG level.
        #
        #    This method does not check the 'layer' part of the flag for
        #    validity.
        #
        #    If used, "-debug" turns on file logging, regardless of any
        #    other settings.
        #
        elif option == '-debug':
            if module_name not in config.log:
                config.log.append(module_name)
            if value not in config.debug_log:
                config.debug_log.append(value)  # may be empty string
        elif option in ('-verbose', '-v'):
            config.verbose_output += 1
        elif option == '-daemonize':
            redirect_std = value or None
            daemonize.daemonize(redirect_std=redirect_std)
        else:
            # the argument depends on numerical config settings
            # e.g. -maxlag and -step:
            try:
                _arg = option[1:]
                # explicitly check for int (so bool doesn't match)
                if not isinstance(getattr(config, _arg), int):
                    raise TypeError
                setattr(config, _arg, int(value))
            except (ValueError, TypeError, AttributeError):
                # argument not global -> specific bot script will take care
                non_global_args.append(arg)

    if calledModuleName() != 'generate_user_files':  # T261771
        try:
            pywikibot.Site()
        except (UnknownFamily, UnknownSite):
            pywikibot.exception()
            sys.exit(1)

    if username:
        config.usernames[config.family][config.mylang] = username

    init_handlers()
    writeToCommandLogFile()

    if config.verbose_output:
        pywikibot.output('Python ' + sys.version)

    if do_help:
        show_help(show_global=do_help == 'global')
        sys.exit(0)

    debug('handle_args() completed.', _logger)
    return non_global_args


def show_help(module_name=None, show_global=False):
    """Show help for the Bot."""
    if not module_name:
        module_name = calledModuleName()
    if not module_name:
        try:
            module_name = sys.modules['__main__'].main.__module__
        except NameError:
            module_name = 'no_module'

    try:
        module = import_module(module_name)
        help_text = module.__doc__
        if hasattr(module, 'docuReplacements'):
            for key, value in module.docuReplacements.items():
                help_text = help_text.replace(key, value.strip('\n\r'))
    except Exception:
        if module_name:
            pywikibot.stdout('Sorry, no help available for ' + module_name)
        pywikibot.log('show_help:', exc_info=True)
    else:
        pywikibot.stdout(help_text)  # output to STDOUT

    if show_global or module_name == 'pwb':
        pywikibot.stdout(_GLOBAL_HELP.format(module_name))
    else:
        pywikibot.stdout(_GLOBAL_HELP_NOTE)


@deprecated('show_help', since='20200705')
def showHelp(module_name=None):  # pragma: no cover
    """DEPRECATED. Use show_help()."""
    return show_help(module_name)


def suggest_help(missing_parameters: Optional[List[str]] = None,
                 missing_generator: bool = False,
                 unknown_parameters: Optional[List[str]] = None,
                 exception=None,
                 missing_action: bool = False,
                 additional_text: str = '',
                 missing_dependencies: Optional[List[str]] = None) -> bool:
    """
    Output error message to use -help with additional text before it.

    @param missing_parameters: A list of parameters which are missing.
    @param missing_generator: Whether a generator is missing.
    @param unknown_parameters: A list of parameters which are unknown.
    @param exception: An exception thrown.
    @type exception: Exception
    @param missing_action: Add an entry that no action was defined.
    @type missing_action: bool
    @param additional_text: Additional text added to the end.
    @param missing_dependencies: A list of dependencies which cannot be
        imported.
    @type missing_dependencies: list of str
    @return: True if an error message was printed, False otherwise
    """
    messages = []
    if exception:
        messages.append('An error occurred: "{}".'.format(exception))
    if missing_generator:
        messages.append(
            'Unable to execute script because no generator was defined.')
    if missing_parameters:
        messages.append('Missing parameter{s} "{params}".'
                        .format(s='s' if len(missing_parameters) > 1 else '',
                                params='", "'.join(missing_parameters)))
    if missing_action:
        messages.append('No action defined.')
    if unknown_parameters:
        messages.append('Unknown parameter{s} "{params}".'
                        .format(s='s' if len(unknown_parameters) > 1 else '',
                                params='", "'.join(unknown_parameters)))
    if missing_dependencies:
        messages.append('Missing dependenc{s} "{deps}".'
                        .format(
                            s='ies' if len(missing_dependencies) > 1 else 'y',
                            deps='", "'.join(missing_dependencies)))
    if additional_text:
        messages.append(additional_text.strip())
    if messages:
        messages.append('Use -help for further information.')
        error('\n'.join(messages))
        return True
    return False


def writeToCommandLogFile():
    """
    Save name of the called module along with all params to logs/commands.log.

    This can be used by user later to track errors or report bugs.
    """
    modname = calledModuleName()
    # put quotation marks around all parameters
    args = [modname] + ['"%s"' % s for s in pywikibot.argvu[1:]]
    command_log_filename = config.datafilepath('logs', 'commands.log')
    try:
        command_log_file = codecs.open(command_log_filename, 'a', 'utf-8')
    except IOError:
        command_log_file = codecs.open(command_log_filename, 'w', 'utf-8')

    with closing(command_log_file):
        # add a timestamp in ISO 8601 formulation
        iso_date = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())
        command_log_file.write('{} r{} Python {} '
                               .format(iso_date,
                                       version.getversiondict()['rev'],
                                       sys.version.split()[0]))
        command_log_file.write(' '.join(args) + os.linesep)


def open_webbrowser(page):
    """Open the web browser displaying the page and wait for input."""
    webbrowser.open(page.full_url())
    i18n.input('pywikibot-enter-finished-browser')


class _OptionDict(dict):

    """The option dict which holds the options of OptionHandler."""

    def __init__(self, classname, options):
        self._classname = classname
        self._options = {}
        super().__init__(options)

    def __missing__(self, key):
        raise pywikibot.Error("'{}' is not a valid option for {}."
                              .format(key, self._classname))

    def __getattr__(self, name):
        """Get item from dict."""
        return self.__getitem__(name)

    def __setattr__(self, name, value):
        """Set item or attribute."""
        if name not in ('_classname', '_options'):
            self.__setitem__(name, value)
        else:
            super().__setattr__(name, value)

    def __getitem__(self, key):
        """Update options fro backward compatibility and get item."""
        self.update_options()
        return super().__getitem__(key)

    def __setitem__(self, key, value):
        """Set item in dict and deprecated option dict."""
        if key in self._options:
            self._options[key] = value
        super().__setitem__(key, value)

    def update_options(self):
        """Update dict from deprecated options for backward compatibility."""
        self.update(self._options)


_DEPRECATION_MSG = 'Optionhandler.opt.option attribute ' \
                   'or Optionhandler.opt[option] item'


class OptionHandler:

    """Class to get and set options.

    How to use options of OptionHandler and its BaseBot subclasses:
    First define an available_options class attribute to define all
    available options:

    available_options = {'foo': 'bar', 'bar': 42, 'baz': False}

    Or you may update the predefined setting in the class initializer.
    BaseBot predefines a 'always' options and sets it to False:

    self.available_options.update(always=True, another_option='Yes')

    Now you can instantiate an OptionHandler or BaseBot class passing
    options other than default values:

    >>> bot = OptionHandler(baz=True)

    You can access bot options either as keyword item or attribute:

    >>> bot.opt.foo
    'bar'
    >>> bot.opt['bar']
    42
    >>> bot.opt.baz  # default was overridden
    True

    You can set the options in the same way:

    >>> bot.opt.bar = 4711
    >>> bot.opt['baz'] = None
    >>>

    Or you can use the option as a dict:

    >>> 'Option opt.{foo} is {bar}'.format_map(bot.opt)<commit_msg>[doctest] Solve doctest for OptionHandler<commit_after>"""
User-interface related functions for building bots.

This module supports several different bot classes which could be used in
conjunction. Each bot should subclass at least one of these four classes:

* L{BaseBot}: Basic bot class in case where the site is handled differently,
  like working on two sites in parallel.

* L{SingleSiteBot}: Bot class which should only be run on a single site. They
  usually store site specific content and thus can't be easily run when the
  generator returns a page on another site. It has a property C{site} which
  can also be changed. If the generator returns a page of a different site
  it'll skip that page.

* L{MultipleSitesBot}: Bot class which supports to be run on multiple sites
  without the need to manually initialize it every time. It is not possible to
  set the C{site} property and it's deprecated to request it. Instead site of
  the current page should be used. And out of C{run} that sit isn't defined.

* L{ConfigParserBot}: Bot class which supports reading options from a
  scripts.ini configuration file. That file consists of sections, led by a
  C{[section]} header and followed by C{option: value} or C{option=value}
  entries. The section is the script name without .py suffix. All options
  identified must be predefined in available_options dictionary.

* L{Bot}: The previous base class which should be avoided. This class is mainly
  used for bots which work with wikibase or together with an image repository.

Additionally there is the L{CurrentPageBot} class which automatically sets the
current page to the page treated. It is recommended to use this class and to
use C{treat_page} instead of C{treat} and C{put_current} instead of C{userPut}.
It by default subclasses the C{BaseBot} class.

With L{CurrentPageBot} it's possible to subclass one of the following classes
to filter the pages which are ultimately handled by C{treat_page}:

* L{ExistingPageBot}: Only handle pages which do exist.
* L{CreatingPageBot}: Only handle pages which do not exist.
* L{RedirectPageBot}: Only handle pages which are redirect pages.
* L{NoRedirectPageBot}: Only handle pages which are not redirect pages.
* L{FollowRedirectPageBot}: If the generator returns a redirect page it'll
  follow the redirect and instead work on the redirected class.

It is possible to combine filters by subclassing multiple of them. They are
new-style classes so when a class is first subclassing L{ExistingPageBot} and
then L{FollowRedirectPageBot} it will also work on pages which do not exist
when a redirect pointed to that. If the order is inversed it'll first follow
them and then check whether they exist.

Additionally there is the L{AutomaticTWSummaryBot} which subclasses
L{CurrentPageBot} and automatically defines the summary when C{put_current} is
used.
"""
#
# (C) Pywikibot team, 2008-2021
#
# Distributed under the terms of the MIT license.
#
__all__ = (
    'CRITICAL', 'ERROR', 'INFO', 'WARNING', 'DEBUG', 'INPUT', 'STDOUT',
    'VERBOSE', 'critical', 'debug', 'error', 'exception', 'log', 'warning',
    'output', 'stdout', 'LoggingFormatter', 'RotatingFileHandler',
    'init_handlers', 'writelogheader',
    'input', 'input_choice', 'input_yn', 'input_list_choice', 'ui',
    'Option', 'StandardOption', 'NestedOption', 'IntegerOption',
    'ContextOption', 'ListOption', 'ShowingListOption', 'MultipleChoiceList',
    'ShowingMultipleChoiceList', 'OutputProxyOption',
    'HighlightContextOption', 'ChoiceException', 'UnhandledAnswer',
    'Choice', 'StaticChoice', 'LinkChoice', 'AlwaysChoice',
    'QuitKeyboardInterrupt',
    'InteractiveReplace',
    'calledModuleName', 'handle_args',
    'show_help', 'showHelp', 'suggest_help',
    'writeToCommandLogFile', 'open_webbrowser',
    'OptionHandler',
    'BaseBot', 'Bot', 'ConfigParserBot', 'SingleSiteBot', 'MultipleSitesBot',
    'CurrentPageBot', 'AutomaticTWSummaryBot',
    'ExistingPageBot', 'FollowRedirectPageBot', 'CreatingPageBot',
    'RedirectPageBot', 'NoRedirectPageBot',
    'WikidataBot',
)


import codecs
import configparser
import datetime
import json
import logging
import logging.handlers
import os
import sys
import time
import warnings
import webbrowser

from contextlib import closing
from importlib import import_module
from pathlib import Path
from textwrap import fill
from typing import Any, Optional, Union
from warnings import warn

import pywikibot
from pywikibot.backports import Dict, Iterable, List, Sequence
from pywikibot import config2 as config
from pywikibot import daemonize
from pywikibot import i18n
from pywikibot import version
from pywikibot.bot_choice import (
    Option, StandardOption, NestedOption, IntegerOption, ContextOption,
    ListOption, ShowingListOption, MultipleChoiceList,
    ShowingMultipleChoiceList, OutputProxyOption, HighlightContextOption,
    ChoiceException, QuitKeyboardInterrupt,
    Choice, StaticChoice, LinkChoice, AlwaysChoice
)
from pywikibot.exceptions import UnknownFamily, UnknownSite
from pywikibot.logging import CRITICAL, ERROR, INFO, WARNING
from pywikibot.logging import DEBUG, INPUT, STDOUT, VERBOSE
from pywikibot.logging import (
    add_init_routine,
    critical, debug, error, exception, log, output, stdout, warning,
)
from pywikibot.tools import (
    deprecated, deprecate_arg, deprecated_args, issue_deprecation_warning,
    PYTHON_VERSION, suppress_warnings
)
from pywikibot.tools._logging import LoggingFormatter, RotatingFileHandler
from pywikibot.tools.formatter import color_format


# Note: all output goes through python std library "logging" module
_logger = 'bot'

# User interface initialization
# search for user interface module in the 'userinterfaces' subdirectory
uiModule = __import__('pywikibot.userinterfaces.%s_interface'
                      % config.userinterface,
                      fromlist=['UI'])
ui = uiModule.UI()
pywikibot.argvu = ui.argvu()

_GLOBAL_HELP = """
GLOBAL OPTIONS
==============
(Global arguments available for all bots)

-dir:PATH         Read the bot's configuration data from directory given by
                  PATH, instead of from the default directory.

-lang:xx          Set the language of the wiki you want to work on, overriding
                  the configuration in user-config.py. xx should be the
                  site code.

-family:xyz       Set the family of the wiki you want to work on, e.g.
                  wikipedia, wiktionary, wikivoyage, ...
                  This will override the configuration in user-config.py.

-site:xyz:xx      Set the wiki site you want to work on, e.g.
                  wikipedia:test, wiktionary:de, wikivoyage:en, ...
                  This will override the configuration in user-config.py.

-user:xyz         Log in as user 'xyz' instead of the default username.

-daemonize:xyz    Immediately return control to the terminal and redirect
                  stdout and stderr to file xyz.
                  (only use for bots that require no input from stdin).

-help             Show this help text.

-log              Enable the log file, using the default filename
                  '{}-bot.log'
                  Logs will be stored in the logs subdirectory.

-log:xyz          Enable the log file, using 'xyz' as the filename.

-nolog            Disable the log file (if it is enabled by default).

-maxlag           Sets a new maxlag parameter to a number of seconds. Defer bot
                  edits during periods of database server lag. Default is set
                  by config.py

-putthrottle:n    Set the minimum time (in seconds) the bot will wait between
-pt:n             saving pages.
-put_throttle:n

-debug:item       Enable the log file and include extensive debugging data
-debug            for component "item" (for all components if the second form
                  is used).

-verbose          Have the bot provide additional console output that may be
-v                useful in debugging.

-cosmeticchanges  Toggles the cosmetic_changes setting made in config.py or
-cc               user-config.py to its inverse and overrules it. All other
                  settings and restrictions are untouched.

-simulate         Disables writing to the server. Useful for testing and
                  debugging of new code (if given, doesn't do any real
                  changes, but only shows what would have been changed).
                  An integer or float value may be given to simulate a
                  processing time; the bot just waits for given seconds.

-<config var>:n   You may use all given numeric config variables as option and
                  modify it with command line.

"""

_GLOBAL_HELP_NOTE = """
GLOBAL OPTIONS
==============
For global options use -help:global or run pwb.py -help

"""


class UnhandledAnswer(Exception):

    """The given answer didn't suffice."""

    def __init__(self, stop=False):
        """Initializer."""
        self.stop = stop


# Initialize the handlers and formatters for the logging system.
#
# This relies on the global variable 'ui' which is a UserInterface object
# defined in the 'userinterface' subpackage.
#
# The UserInterface object must define its own init_handlers() method
# which takes the root logger as its only argument, and which adds to that
# logger whatever handlers and formatters are needed to process output and
# display it to the user. The default (terminal) interface sends level
# STDOUT to sys.stdout (as all interfaces should) and sends all other
# levels to sys.stderr; levels WARNING and above are labeled with the
# level name.
#
# UserInterface objects must also define methods input(), input_choice(),
# and editText(), all of which are documented in
# userinterfaces/terminal_interface.py

_handlers_initialized = False


def init_handlers(strm=None):
    """Initialize logging system for terminal-based bots.

    This function must be called before using pywikibot.output(); and must
    be called again if the destination stream is changed.

    Note: this function is called by handle_args(), so it should normally
    not need to be called explicitly

    All user output is routed through the logging module.
    Each type of output is handled by an appropriate handler object.
    This structure is used to permit eventual development of other
    user interfaces (GUIs) without modifying the core bot code.

    The following output levels are defined:

     - DEBUG: only for file logging; debugging messages.
     - STDOUT: output that must be sent to sys.stdout (for bots that may
       have their output redirected to a file or other destination).
     - VERBOSE: optional progress information for display to user.
     - INFO: normal (non-optional) progress information for display to user.
     - INPUT: prompts requiring user response.
     - WARN: user warning messages.
     - ERROR: user error messages.
     - CRITICAL: fatal error messages.

    Accordingly, do **not** use print statements in bot code; instead,
    use pywikibot.output function.

    @param strm: Output stream. If None, re-uses the last stream if one
        was defined, otherwise uses sys.stderr

    """
    global _handlers_initialized

    module_name = calledModuleName()
    if not module_name:
        module_name = 'terminal-interface'

    logging.addLevelName(VERBOSE, 'VERBOSE')
    # for messages to be displayed on terminal at "verbose" setting
    # use INFO for messages to be displayed even on non-verbose setting

    logging.addLevelName(STDOUT, 'STDOUT')
    # for messages to be displayed to stdout

    logging.addLevelName(INPUT, 'INPUT')
    # for prompts requiring user response

    root_logger = logging.getLogger('pywiki')
    root_logger.setLevel(DEBUG + 1)  # all records except DEBUG go to logger

    warnings_logger = logging.getLogger('py.warnings')
    warnings_logger.setLevel(DEBUG)

    # If there are command line warnings options, do not override them
    if not sys.warnoptions:
        logging.captureWarnings(True)

        if config.debug_log or 'deprecation' in config.log:
            warnings.filterwarnings('always')
        elif config.verbose_output:
            warnings.filterwarnings('module')
        warnings.filterwarnings('once', category=FutureWarning)

    root_logger.handlers = []  # remove any old handlers

    # configure handler(s) for display to user interface
    ui.init_handlers(root_logger, **config.userinterface_init_kwargs)

    # if user has enabled file logging, configure file handler
    if module_name in config.log or '*' in config.log:
        if config.logfilename:
            logfile = config.datafilepath('logs', config.logfilename)
        else:
            logfile = config.datafilepath('logs', '%s-bot.log' % module_name)
        file_handler = RotatingFileHandler(filename=logfile,
                                           maxBytes=1024 * config.logfilesize,
                                           backupCount=config.logfilecount,
                                           encoding='utf-8')

        file_handler.setLevel(DEBUG)
        form = LoggingFormatter(
            fmt='%(asctime)s %(caller_file)18s, %(caller_line)4s '
                'in %(caller_name)18s: %(levelname)-8s %(message)s',
            datefmt='%Y-%m-%d %H:%M:%S'
        )
        file_handler.setFormatter(form)
        root_logger.addHandler(file_handler)
        # Turn on debugging for each component requested by user
        # or for all components if nothing was specified
        for component in config.debug_log:
            if component:
                debuglogger = logging.getLogger('pywiki.' + component)
            else:
                debuglogger = logging.getLogger('pywiki')
            debuglogger.setLevel(DEBUG)
            debuglogger.addHandler(file_handler)

        warnings_logger.addHandler(file_handler)

    _handlers_initialized = True

    writelogheader()


def writelogheader():
    """
    Save additional version, system and status info to the log file in use.

    This may help the user to track errors or report bugs.
    """
    log('')
    log('=== Pywikibot framework v{} -- Logging header ==='
        .format(pywikibot.__version__))

    # script call
    log('COMMAND: {}'.format(sys.argv))

    # script call time stamp
    log('DATE: {} UTC'.format(datetime.datetime.utcnow()))

    # new framework release/revision? (handle_args needs to be called first)
    try:
        log('VERSION: {}'.format(
            version.getversion(online=config.log_pywiki_repo_version).strip()))
    except version.ParseError:
        exception()

    # system
    if hasattr(os, 'uname'):
        log('SYSTEM: {}'.format(os.uname()))

    # config file dir
    log('CONFIG FILE DIR: {}'.format(pywikibot.config2.base_dir))

    all_modules = sys.modules.keys()

    # These are the main dependencies of pywikibot.
    check_package_list = [
        'requests',
        'mwparserfromhell',
    ]

    # report all imported packages
    if config.verbose_output:
        check_package_list += all_modules

    log('PACKAGES:')
    packages = version.package_versions(check_package_list)
    for name in sorted(packages.keys()):
        info = packages[name]
        info.setdefault('path',
                        '[{}]'.format(info.get('type', 'path unknown')))
        info.setdefault('ver', '??')
        if 'err' in info:
            log('  {name}: {err}'.format_map(info))
        else:
            log('  {name} ({path}) = {ver}'.format_map(info))

    # imported modules
    log('MODULES:')
    for module in sys.modules.values():
        filename = version.get_module_filename(module)
        if not filename:
            continue

        param = {'sep': ' '}
        if PYTHON_VERSION >= (3, 6, 0):
            param['timespec'] = 'seconds'
        mtime = version.get_module_mtime(module).isoformat(**param)

        log('  {} {}'
            .format(mtime, filename))

    if config.log_pywiki_repo_version:
        log('PYWIKI REPO VERSION: {}'.format(version.getversion_onlinerepo()))

    log('=' * 57)


add_init_routine(init_handlers)


# User input functions


def input(question: str, password: bool = False,
          default: str = '', force: bool = False) -> str:
    """Ask the user a question, return the user's answer.

    @param question: a string that will be shown to the user. Don't add a
        space after the question mark/colon, this method will do this for you.
    @param password: if True, hides the user's input (for password entry).
    @param default: The default answer if none was entered. None to require
        an answer.
    @param force: Automatically use the default
    """
    # make sure logging system has been initialized
    if not _handlers_initialized:
        init_handlers()

    data = ui.input(question, password=password, default=default, force=force)
    return data


def input_choice(question: str, answers, default: Optional[str] = None,
                 return_shortcut: bool = True,
                 automatic_quit: bool = True, force: bool = False):
    """
    Ask the user the question and return one of the valid answers.

    @param question: The question asked without trailing spaces.
    @param answers: The valid answers each containing a full length answer and
        a shortcut. Each value must be unique.
    @type answers: iterable containing a sequence of length two or instances of
        ChoiceException
    @param default: The result if no answer was entered. It must not be in the
        valid answers and can be disabled by setting it to None. If it should
        be linked with the valid answers it must be its shortcut.
    @param return_shortcut: Whether the shortcut or the index of the answer is
        returned.
    @param automatic_quit: Adds the option 'Quit' ('q') and throw a
        L{QuitKeyboardInterrupt} if selected.
    @param force: Automatically use the default
    @return: The selected answer shortcut or index. Is -1 if the default is
        selected, it does not return the shortcut and the default is not a
        valid shortcut.
    @rtype: int (if not return shortcut), str (otherwise)
    """
    # make sure logging system has been initialized
    if not _handlers_initialized:
        init_handlers()

    return ui.input_choice(question, answers, default, return_shortcut,
                           automatic_quit=automatic_quit, force=force)


def input_yn(question: str,
             default: Union[str, bool, None] = None,
             automatic_quit: bool = True,
             force: bool = False) -> bool:
    """
    Ask the user a yes/no question and return the answer as a bool.

    @param question: The question asked without trailing spaces.
    @param default: The result if no answer was entered. It must be a bool or
        'y' or 'n' and can be disabled by setting it to None.
    @param automatic_quit: Adds the option 'Quit' ('q') and throw a
        L{QuitKeyboardInterrupt} if selected.
    @param force: Automatically use the default
    @return: Return True if the user selected yes and False if the user
        selected no. If the default is not None it'll return True if default
        is True or 'y' and False if default is False or 'n'.
    """
    if default not in ['y', 'Y', 'n', 'N']:
        if default:
            default = 'y'
        elif default is not None:
            default = 'n'
    assert default in ['y', 'Y', 'n', 'N', None], \
        'Default choice must be one of YyNn or default'

    return input_choice(question, [('Yes', 'y'), ('No', 'n')], default,
                        automatic_quit=automatic_quit, force=force) == 'y'


def input_list_choice(question: str, answers: Sequence[Any],
                      default: Union[int, str, None] = None,
                      force: bool = False) -> str:
    """
    Ask the user the question and return one of the valid answers.

    @param question: The question asked without trailing spaces.
    @param answers: The valid answers each containing a full length answer.
    @param default: The result if no answer was entered. It must not be in the
        valid answers and can be disabled by setting it to None.
    @param force: Automatically use the default
    @return: The selected answer.
    """
    if not _handlers_initialized:
        init_handlers()

    return ui.input_list_choice(question, answers, default=default,
                                force=force)


class InteractiveReplace:

    """
    A callback class for textlib's replace_links.

    It shows various options which can be switched on and off:
    * allow_skip_link = True (skip the current link)
    * allow_unlink = True (unlink)
    * allow_replace = False (just replace target, keep section and label)
    * allow_replace_section = False (replace target and section, keep label)
    * allow_replace_label = False (replace target and label, keep section)
    * allow_replace_all = False (replace target, section and label)
    (The boolean values are the default values)

    It has also a C{context} attribute which must be a non-negative integer. If
    it is greater 0 it shows that many characters before and after the link in
    question. The C{context_delta} attribute can be defined too and adds an
    option to increase C{context} by the given amount each time the option is
    selected.

    Additional choices can be defined using the 'additional_choices' and will
    be amended to the choices defined by this class. This list is mutable and
    the Choice instance returned and created by this class are too.
    """

    def __init__(self, old_link, new_link, default: Optional[str] = None,
                 automatic_quit: bool = True):
        """
        Initializer.

        @param old_link: The old link which is searched. The label and section
            are ignored.
        @type old_link: pywikibot.page.Link or pywikibot.page.Page
        @param new_link: The new link with which it should be replaced.
            Depending on the replacement mode it'll use this link's label and
            section. If False it'll unlink all and the attributes beginning
            with allow_replace are ignored.
        @type new_link: pywikibot.page.Link or pywikibot.page.Page or False
        @param default: The default answer as the shortcut
        @param automatic_quit: Add an option to quit and raise a
            QuitKeyboardException.
        """
        if isinstance(old_link, pywikibot.Page):
            self._old = old_link._link
        else:
            self._old = old_link
        if isinstance(new_link, pywikibot.Page):
            self._new = new_link._link
        else:
            self._new = new_link
        self._default = default
        self._quit = automatic_quit
        self._current_match = None
        self.context = 30
        self.context_delta = 0
        self.allow_skip_link = True
        self.allow_unlink = True
        self.allow_replace = False
        self.allow_replace_section = False
        self.allow_replace_label = False
        self.allow_replace_all = False
        # Use list to preserve order
        self._own_choices = [
            ('skip_link', StaticChoice('Do not change', 'n', None)),
            ('unlink', StaticChoice('Unlink', 'u', False)),
        ]
        if self._new:
            self._own_choices += [
                ('replace', LinkChoice('Change link target', 't', self,
                                       False, False)),
                ('replace_section', LinkChoice(
                    'Change link target and section', 's', self, True, False)),
                ('replace_label', LinkChoice('Change link target and label',
                                             'l', self, False, True)),
                ('replace_all', LinkChoice('Change complete link', 'c', self,
                                           True, True)),
            ]

        self.additional_choices = []

    def handle_answer(self, choice):
        """Return the result for replace_links."""
        for c in self.choices:
            if c.shortcut == choice:
                return c.handle()
        else:
            raise ValueError('Invalid choice "{0}"'.format(choice))

    def __call__(self, link, text, groups, rng):
        """Ask user how the selected link should be replaced."""
        if self._old == link:
            self._current_match = (link, text, groups, rng)
            while True:
                try:
                    answer = self.handle_link()
                except UnhandledAnswer as e:
                    if e.stop:
                        raise
                else:
                    break
            self._current_match = None  # don't reset in case of an exception
            return answer
        else:
            return None

    @property
    def choices(self):
        """Return the tuple of choices."""
        choices = []
        for name, choice in self._own_choices:
            if getattr(self, 'allow_' + name):
                choices += [choice]
        if self.context_delta > 0:
            choices += [HighlightContextOption(
                'more context', 'm', self.current_text, self.context,
                self.context_delta, *self.current_range)]
        choices += self.additional_choices
        return tuple(choices)

    def handle_link(self):
        """Handle the currently given replacement."""
        choices = self.choices
        for choice in choices:
            if isinstance(choice, Choice) and choice.handle_link():
                return choice.answer

        if self.context > 0:
            rng = self.current_range
            text = self.current_text
            # at the beginning of the link, start red color.
            # at the end of the link, reset the color to default
            pywikibot.output(text[max(0, rng[0] - self.context): rng[0]]
                             + color_format('{lightred}{0}{default}',
                                            text[rng[0]: rng[1]])
                             + text[rng[1]: rng[1] + self.context])
            question = 'Should the link '
        else:
            question = 'Should the link {lightred}{0}{default} '

        if self._new is False:
            question += 'be unlinked?'
        else:
            question += color_format('target to {lightpurple}{0}{default}?',
                                     self._new.canonical_title())

        choice = pywikibot.input_choice(
            color_format(question, self._old.canonical_title()),
            choices, default=self._default, automatic_quit=self._quit)

        return self.handle_answer(choice)

    @property
    def current_link(self):
        """Get the current link when it's handling one currently."""
        if self._current_match is None:
            raise ValueError('No current link')
        return self._current_match[0]

    @property
    def current_text(self):
        """Get the current text when it's handling one currently."""
        if self._current_match is None:
            raise ValueError('No current text')
        return self._current_match[1]

    @property
    def current_groups(self):
        """Get the current groups when it's handling one currently."""
        if self._current_match is None:
            raise ValueError('No current groups')
        return self._current_match[2]

    @property
    def current_range(self):
        """Get the current range when it's handling one currently."""
        if self._current_match is None:
            raise ValueError('No current range')
        return self._current_match[3]


# Command line parsing and help
def calledModuleName() -> str:
    """Return the name of the module calling this function.

    This is required because the -help option loads the module's docstring
    and because the module name will be used for the filename of the log.

    """
    return Path(pywikibot.argvu[0]).stem


def handle_args(args: Optional[Iterable[str]] = None,
                do_help: bool = True) -> List[str]:
    """
    Handle standard command line arguments, and return the rest as a list.

    Takes the command line arguments as strings, processes all global
    parameters such as -lang or -log, initialises the logging layer,
    which emits startup information into log at level 'verbose'.

    This makes sure that global arguments are applied first,
    regardless of the order in which the arguments were given.

    args may be passed as an argument, thereby overriding sys.argv

    @param args: Command line arguments
    @param do_help: Handle parameter '-help' to show help and invoke sys.exit
    @return: list of arguments not recognised globally
    """
    if pywikibot._sites:
        warn('Site objects have been created before arguments were handled',
             UserWarning)

    # get commandline arguments if necessary
    if not args:
        # it's the version in pywikibot.__init__ that is changed by scripts,
        # not the one in pywikibot.bot.
        args = pywikibot.argvu[1:]

    # get the name of the module calling this function. This is
    # required because the -help option loads the module's docstring and
    # because the module name will be used for the filename of the log.
    module_name = calledModuleName() or 'terminal-interface'
    non_global_args = []
    username = None
    do_help = None if do_help else False
    for arg in args:
        option, _, value = arg.partition(':')
        if do_help is not False and option == '-help':
            do_help = value or True
        elif option == '-dir':
            pass
        elif option == '-site':
            config.family, config.mylang = value.split(':')
        elif option == '-family':
            config.family = value
        elif option == '-lang':
            config.mylang = value
        elif option == '-user':
            username = value
        elif option in ('-putthrottle', '-pt'):
            config.put_throttle = float(value)
        elif option == '-log':
            if module_name not in config.log:
                config.log.append(module_name)
            if value:
                config.logfilename = value
        elif option == '-nolog':
            config.log = []
        elif option in ('-cosmeticchanges', '-cc'):
            config.cosmetic_changes = not config.cosmetic_changes
            output('NOTE: option cosmetic_changes is %s\n'
                   % config.cosmetic_changes)
        elif option == '-simulate':
            config.simulate = value or True
        #
        #  DEBUG control:
        #
        #    The framework has four layers (by default, others can be added),
        #    each designated by a string --
        #
        #    1.  "comm": the communication layer (http requests, etc.)
        #    2.  "data": the raw data layer (API requests, XML dump parsing)
        #    3.  "wiki": the wiki content representation layer (Page and Site
        #         objects)
        #    4.  "bot": the application layer (user scripts should always
        #         send any debug() messages to this layer)
        #
        #    The "-debug:layer" flag sets the logger for any specified
        #    layer to the DEBUG level, causing it to output extensive debugging
        #    information. Otherwise, the default logging setting is the INFO
        #    level. "-debug" with no layer specified sets _all_ loggers to
        #    DEBUG level.
        #
        #    This method does not check the 'layer' part of the flag for
        #    validity.
        #
        #    If used, "-debug" turns on file logging, regardless of any
        #    other settings.
        #
        elif option == '-debug':
            if module_name not in config.log:
                config.log.append(module_name)
            if value not in config.debug_log:
                config.debug_log.append(value)  # may be empty string
        elif option in ('-verbose', '-v'):
            config.verbose_output += 1
        elif option == '-daemonize':
            redirect_std = value or None
            daemonize.daemonize(redirect_std=redirect_std)
        else:
            # the argument depends on numerical config settings
            # e.g. -maxlag and -step:
            try:
                _arg = option[1:]
                # explicitly check for int (so bool doesn't match)
                if not isinstance(getattr(config, _arg), int):
                    raise TypeError
                setattr(config, _arg, int(value))
            except (ValueError, TypeError, AttributeError):
                # argument not global -> specific bot script will take care
                non_global_args.append(arg)

    if calledModuleName() != 'generate_user_files':  # T261771
        try:
            pywikibot.Site()
        except (UnknownFamily, UnknownSite):
            pywikibot.exception()
            sys.exit(1)

    if username:
        config.usernames[config.family][config.mylang] = username

    init_handlers()
    writeToCommandLogFile()

    if config.verbose_output:
        pywikibot.output('Python ' + sys.version)

    if do_help:
        show_help(show_global=do_help == 'global')
        sys.exit(0)

    debug('handle_args() completed.', _logger)
    return non_global_args


def show_help(module_name=None, show_global=False):
    """Show help for the Bot."""
    if not module_name:
        module_name = calledModuleName()
    if not module_name:
        try:
            module_name = sys.modules['__main__'].main.__module__
        except NameError:
            module_name = 'no_module'

    try:
        module = import_module(module_name)
        help_text = module.__doc__
        if hasattr(module, 'docuReplacements'):
            for key, value in module.docuReplacements.items():
                help_text = help_text.replace(key, value.strip('\n\r'))
    except Exception:
        if module_name:
            pywikibot.stdout('Sorry, no help available for ' + module_name)
        pywikibot.log('show_help:', exc_info=True)
    else:
        pywikibot.stdout(help_text)  # output to STDOUT

    if show_global or module_name == 'pwb':
        pywikibot.stdout(_GLOBAL_HELP.format(module_name))
    else:
        pywikibot.stdout(_GLOBAL_HELP_NOTE)


@deprecated('show_help', since='20200705')
def showHelp(module_name=None):  # pragma: no cover
    """DEPRECATED. Use show_help()."""
    return show_help(module_name)


def suggest_help(missing_parameters: Optional[List[str]] = None,
                 missing_generator: bool = False,
                 unknown_parameters: Optional[List[str]] = None,
                 exception=None,
                 missing_action: bool = False,
                 additional_text: str = '',
                 missing_dependencies: Optional[List[str]] = None) -> bool:
    """
    Output error message to use -help with additional text before it.

    @param missing_parameters: A list of parameters which are missing.
    @param missing_generator: Whether a generator is missing.
    @param unknown_parameters: A list of parameters which are unknown.
    @param exception: An exception thrown.
    @type exception: Exception
    @param missing_action: Add an entry that no action was defined.
    @type missing_action: bool
    @param additional_text: Additional text added to the end.
    @param missing_dependencies: A list of dependencies which cannot be
        imported.
    @type missing_dependencies: list of str
    @return: True if an error message was printed, False otherwise
    """
    messages = []
    if exception:
        messages.append('An error occurred: "{}".'.format(exception))
    if missing_generator:
        messages.append(
            'Unable to execute script because no generator was defined.')
    if missing_parameters:
        messages.append('Missing parameter{s} "{params}".'
                        .format(s='s' if len(missing_parameters) > 1 else '',
                                params='", "'.join(missing_parameters)))
    if missing_action:
        messages.append('No action defined.')
    if unknown_parameters:
        messages.append('Unknown parameter{s} "{params}".'
                        .format(s='s' if len(unknown_parameters) > 1 else '',
                                params='", "'.join(unknown_parameters)))
    if missing_dependencies:
        messages.append('Missing dependenc{s} "{deps}".'
                        .format(
                            s='ies' if len(missing_dependencies) > 1 else 'y',
                            deps='", "'.join(missing_dependencies)))
    if additional_text:
        messages.append(additional_text.strip())
    if messages:
        messages.append('Use -help for further information.')
        error('\n'.join(messages))
        return True
    return False


def writeToCommandLogFile():
    """
    Save name of the called module along with all params to logs/commands.log.

    This can be used by user later to track errors or report bugs.
    """
    modname = calledModuleName()
    # put quotation marks around all parameters
    args = [modname] + ['"%s"' % s for s in pywikibot.argvu[1:]]
    command_log_filename = config.datafilepath('logs', 'commands.log')
    try:
        command_log_file = codecs.open(command_log_filename, 'a', 'utf-8')
    except IOError:
        command_log_file = codecs.open(command_log_filename, 'w', 'utf-8')

    with closing(command_log_file):
        # add a timestamp in ISO 8601 formulation
        iso_date = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())
        command_log_file.write('{} r{} Python {} '
                               .format(iso_date,
                                       version.getversiondict()['rev'],
                                       sys.version.split()[0]))
        command_log_file.write(' '.join(args) + os.linesep)


def open_webbrowser(page):
    """Open the web browser displaying the page and wait for input."""
    webbrowser.open(page.full_url())
    i18n.input('pywikibot-enter-finished-browser')


class _OptionDict(dict):

    """The option dict which holds the options of OptionHandler."""

    def __init__(self, classname, options):
        self._classname = classname
        self._options = {}
        super().__init__(options)

    def __missing__(self, key):
        raise pywikibot.Error("'{}' is not a valid option for {}."
                              .format(key, self._classname))

    def __getattr__(self, name):
        """Get item from dict."""
        return self.__getitem__(name)

    def __setattr__(self, name, value):
        """Set item or attribute."""
        if name not in ('_classname', '_options'):
            self.__setitem__(name, value)
        else:
            super().__setattr__(name, value)

    def __getitem__(self, key):
        """Update options fro backward compatibility and get item."""
        self.update_options()
        return super().__getitem__(key)

    def __setitem__(self, key, value):
        """Set item in dict and deprecated option dict."""
        if key in self._options:
            self._options[key] = value
        super().__setitem__(key, value)

    def update_options(self):
        """Update dict from deprecated options for backward compatibility."""
        self.update(self._options)


_DEPRECATION_MSG = 'Optionhandler.opt.option attribute ' \
                   'or Optionhandler.opt[option] item'


class OptionHandler:

    """Class to get and set options.

    How to use options of OptionHandler and its BaseBot subclasses:
    First define an available_options class attribute for your own
    option handler to define all available options:

    >>> default_options = {'foo': 'bar', 'bar': 42, 'baz': False}
    >>> class MyHandler(OptionHandler): available_options = default_options

    Or you may update the predefined setting in the class initializer.
    BaseBot predefines a 'always' options and sets it to False:

    self.available_options.update(always=True, another_option='Yes')

    Now you can instantiate an OptionHandler or BaseBot class passing
    options other than default values:

    >>> bot = MyHandler(baz=True)

    You can access bot options either as keyword item or attribute:

    >>> bot.opt.foo
    'bar'
    >>> bot.opt['bar']
    42
    >>> bot.opt.baz  # default was overridden
    True

    You can set the options in the same way:

    >>> bot.opt.bar = 4711
    >>> bot.opt['baz'] = None
    >>>

    Or you can use the option as a dict:

    >>> 'Option opt.{foo} is {bar}'.format_map(bot.opt)
{'fraction_non_alphanumeric': 0.08558602999210733, 'fraction_numerical': 0.0023677979479084454, 'mean_word_length': 2.4914531754574814, 'pattern_counts': {'":': 8, '<': 5, '<?xml version=': 0, '>': 85, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4071185', 'n_tokens_mistral': 21904, 'n_tokens_neox': 21107, 'n_words': 8024}