<commit_before># -*- coding: utf-8  -*-
"""
User-interface related functions for building bots
"""
#
# (C) Pywikipedia bot team, 2008-2011
#
# Distributed under the terms of the MIT license.
#
__version__ = '$Id$'

# Note: the intention is to develop this module (at some point) into a Bot
# class definition that can be subclassed to create new, functional bot
# scripts, instead of writing each one from scratch.


import logging, logging.handlers
       # all output goes thru python std library "logging" module
import os
import os.path
import sys

# logging levels
_logger = "bot"

from logging import DEBUG, INFO, WARNING, ERROR, CRITICAL
STDOUT = 16
VERBOSE = 18
INPUT = 25

import pywikibot
from pywikibot import config


# User interface initialization
# search for user interface module in the 'userinterfaces' subdirectory
uiModule = __import__("pywikibot.userinterfaces.%s_interface"
                        % config.userinterface,
                        fromlist=['UI'] )
ui = uiModule.UI()


# Logging module configuration

class RotatingFileHandler(logging.handlers.RotatingFileHandler):
    """Strip trailing newlines before outputting text to file"""
    def format(self, record):
        text = logging.handlers.RotatingFileHandler.format(self, record)
        return text.rstrip("\r\n")


class LoggingFormatter(logging.Formatter):
    """Format LogRecords for output to file.

    This formatter *ignores* the 'newline' key of the LogRecord, because
    every record written to a file must end with a newline, regardless of
    whether the output to the user's console does.

    """
    def formatException(self, ei):
        """
        Make sure that the exception trace is converted to unicode:
            * our pywikibot.Error traces are encoded in our
              console encoding, which is needed for plainly printing them.
            * but when it comes to logging using logging.exception,
              the Python logging module will try to use these traces,
              and it will fail if they are console encoded strings.

        Formatter.formatException also strips the trailing \n, which we need.
        """
        strExc = logging.Formatter.formatException(self, ei)

        if isinstance(strExc, str):
            return strExc.decode(config.console_encoding) + '\n'
        else:
            return strExc + '\n'


# Initialize the handlers and formatters for the logging system.
#
# This relies on the global variable 'ui' which is a UserInterface object
# defined in the 'userinterface' subpackage.
#
# The UserInterface object must define its own init_handlers() method
# which takes the root logger as its only argument, and which adds to that
# logger whatever handlers and formatters are needed to process output and
# display it to the user.  The default (terminal) interface sends level
# STDOUT to sys.stdout (as all interfaces should) and sends all other
# levels to sys.stderr; levels WARNING and above are labeled with the
# level name.
#
# UserInterface objects must also define methods input(), inputChoice(),
# editText(), and askForCaptcha(), all of which are documented in
# userinterfaces/terminal_interface.py

_handlers_initialized = False

def init_handlers(strm=None):
    """Initialize logging system for terminal-based bots.

    This function must be called before using pywikibot.output(); and must
    be called again if the destination stream is changed.

    @param strm: Output stream. If None, re-uses the last stream if one
        was defined, otherwise uses sys.stderr

    """
    # Note: this function is called by handleArgs(), so it should normally
    # not need to be called explicitly

    # All user output is routed through the logging module.
    # Each type of output is handled by an appropriate handler object.
    # This structure is used to permit eventual development of other
    # user interfaces (GUIs) without modifying the core bot code.
    # The following output levels are defined:
    #    DEBUG - only for file logging; debugging messages
    #    STDOUT - output that must be sent to sys.stdout (for bots that may
    #             have their output redirected to a file or other destination)
    #    VERBOSE - optional progress information for display to user
    #    INFO - normal (non-optional) progress information for display to user
    #    INPUT - prompts requiring user response
    #    WARN - user warning messages
    #    ERROR - user error messages
    #    CRITICAL - fatal error messages
    # Accordingly, do ''not'' use print statements in bot code; instead,
    # use pywikibot.output function.

    global _handlers_initialized

    moduleName = calledModuleName()
    if not moduleName:
        moduleName = "terminal-interface"

    logging.addLevelName(VERBOSE, "VERBOSE")
        # for messages to be displayed on terminal at "verbose" setting
        # use INFO for messages to be displayed even on non-verbose setting
    logging.addLevelName(STDOUT, "STDOUT")
        # for messages to be displayed to stdout
    logging.addLevelName(INPUT, "INPUT")
        # for prompts requiring user response

    root_logger = logging.getLogger("pywiki")
    root_logger.setLevel(DEBUG+1) # all records except DEBUG go to logger
    root_logger.handlers = [] # remove any old handlers

    # configure handler(s) for display to user interface
    ui.init_handlers(root_logger, **config.userinterface_init_kwargs)

    # if user has enabled file logging, configure file handler
    if moduleName in config.log or '*' in config.log:
        if config.logfilename:
            logfile = config.datafilepath(config.logfilename)
        else:
            logfile = config.datafilepath("%s-bot.log" % moduleName)
        file_handler = RotatingFileHandler(
                            filename=logfile, maxBytes=2 << 20, backupCount=5)

        file_handler.setLevel(DEBUG)
        form = LoggingFormatter(
                   fmt="%(asctime)s %(caller_file)18s, %(caller_line)4s "
                       "in %(caller_name)18s: %(levelname)-8s %(message)s",
                   datefmt="%Y-%m-%d %H:%M:%S"
               )
        file_handler.setFormatter(form)
        root_logger.addHandler(file_handler)
        # Turn on debugging for each component requested by user
        # or for all components if nothing was specified
        for component in config.debug_log:
            if component:
                debuglogger = logging.getLogger("pywiki."+component)
            else:
                debuglogger = logging.getLogger("pywiki")<commit_msg>put logs into the logs sub-folder<commit_after># -*- coding: utf-8  -*-
"""
User-interface related functions for building bots
"""
#
# (C) Pywikipedia bot team, 2008-2011
#
# Distributed under the terms of the MIT license.
#
__version__ = '$Id$'

# Note: the intention is to develop this module (at some point) into a Bot
# class definition that can be subclassed to create new, functional bot
# scripts, instead of writing each one from scratch.


import logging, logging.handlers
       # all output goes thru python std library "logging" module
import os
import os.path
import sys

# logging levels
_logger = "bot"

from logging import DEBUG, INFO, WARNING, ERROR, CRITICAL
STDOUT = 16
VERBOSE = 18
INPUT = 25

import pywikibot
from pywikibot import config


# User interface initialization
# search for user interface module in the 'userinterfaces' subdirectory
uiModule = __import__("pywikibot.userinterfaces.%s_interface"
                        % config.userinterface,
                        fromlist=['UI'] )
ui = uiModule.UI()


# Logging module configuration

class RotatingFileHandler(logging.handlers.RotatingFileHandler):
    """Strip trailing newlines before outputting text to file"""
    def format(self, record):
        text = logging.handlers.RotatingFileHandler.format(self, record)
        return text.rstrip("\r\n")


class LoggingFormatter(logging.Formatter):
    """Format LogRecords for output to file.

    This formatter *ignores* the 'newline' key of the LogRecord, because
    every record written to a file must end with a newline, regardless of
    whether the output to the user's console does.

    """
    def formatException(self, ei):
        """
        Make sure that the exception trace is converted to unicode:
            * our pywikibot.Error traces are encoded in our
              console encoding, which is needed for plainly printing them.
            * but when it comes to logging using logging.exception,
              the Python logging module will try to use these traces,
              and it will fail if they are console encoded strings.

        Formatter.formatException also strips the trailing \n, which we need.
        """
        strExc = logging.Formatter.formatException(self, ei)

        if isinstance(strExc, str):
            return strExc.decode(config.console_encoding) + '\n'
        else:
            return strExc + '\n'


# Initialize the handlers and formatters for the logging system.
#
# This relies on the global variable 'ui' which is a UserInterface object
# defined in the 'userinterface' subpackage.
#
# The UserInterface object must define its own init_handlers() method
# which takes the root logger as its only argument, and which adds to that
# logger whatever handlers and formatters are needed to process output and
# display it to the user.  The default (terminal) interface sends level
# STDOUT to sys.stdout (as all interfaces should) and sends all other
# levels to sys.stderr; levels WARNING and above are labeled with the
# level name.
#
# UserInterface objects must also define methods input(), inputChoice(),
# editText(), and askForCaptcha(), all of which are documented in
# userinterfaces/terminal_interface.py

_handlers_initialized = False

def init_handlers(strm=None):
    """Initialize logging system for terminal-based bots.

    This function must be called before using pywikibot.output(); and must
    be called again if the destination stream is changed.

    @param strm: Output stream. If None, re-uses the last stream if one
        was defined, otherwise uses sys.stderr

    """
    # Note: this function is called by handleArgs(), so it should normally
    # not need to be called explicitly

    # All user output is routed through the logging module.
    # Each type of output is handled by an appropriate handler object.
    # This structure is used to permit eventual development of other
    # user interfaces (GUIs) without modifying the core bot code.
    # The following output levels are defined:
    #    DEBUG - only for file logging; debugging messages
    #    STDOUT - output that must be sent to sys.stdout (for bots that may
    #             have their output redirected to a file or other destination)
    #    VERBOSE - optional progress information for display to user
    #    INFO - normal (non-optional) progress information for display to user
    #    INPUT - prompts requiring user response
    #    WARN - user warning messages
    #    ERROR - user error messages
    #    CRITICAL - fatal error messages
    # Accordingly, do ''not'' use print statements in bot code; instead,
    # use pywikibot.output function.

    global _handlers_initialized

    moduleName = calledModuleName()
    if not moduleName:
        moduleName = "terminal-interface"

    logging.addLevelName(VERBOSE, "VERBOSE")
        # for messages to be displayed on terminal at "verbose" setting
        # use INFO for messages to be displayed even on non-verbose setting
    logging.addLevelName(STDOUT, "STDOUT")
        # for messages to be displayed to stdout
    logging.addLevelName(INPUT, "INPUT")
        # for prompts requiring user response

    root_logger = logging.getLogger("pywiki")
    root_logger.setLevel(DEBUG+1) # all records except DEBUG go to logger
    root_logger.handlers = [] # remove any old handlers

    # configure handler(s) for display to user interface
    ui.init_handlers(root_logger, **config.userinterface_init_kwargs)

    # if user has enabled file logging, configure file handler
    if moduleName in config.log or '*' in config.log:
        if config.logfilename:
            logfile = config.datafilepath("logs", config.logfilename)
        else:
            logfile = config.datafilepath("logs", "%s-bot.log" % moduleName)
        file_handler = RotatingFileHandler(
                            filename=logfile, maxBytes=2 << 20, backupCount=5)

        file_handler.setLevel(DEBUG)
        form = LoggingFormatter(
                   fmt="%(asctime)s %(caller_file)18s, %(caller_line)4s "
                       "in %(caller_name)18s: %(levelname)-8s %(message)s",
                   datefmt="%Y-%m-%d %H:%M:%S"
               )
        file_handler.setFormatter(form)
        root_logger.addHandler(file_handler)
        # Turn on debugging for each component requested by user
        # or for all components if nothing was specified
        for component in config.debug_log:
            if component:
                debuglogger = logging.getLogger("pywiki."+component)
            else:
                debuglogger = logging.getLogger("pywiki")
{'fraction_non_alphanumeric': 0.07213288834951456, 'fraction_numerical': 0.003944174757281553, 'mean_word_length': 2.929955290611028, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7439357', 'n_tokens_mistral': 3507, 'n_tokens_neox': 3356, 'n_words': 1400}