codingutf8 2020 optuna hugging face licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license logging utilities import functools import logging import os import sys import threading from logging import critical noqa debug noqa error noqa fatal noqa info noqa notset noqa warn noqa warning noqa from logging import capturewarnings as capturewarnings from typing import optional import huggingfacehub utils as hfhubutils from tqdm import auto as tqdmlib lock threading lock defaulthandler optionallogging handler none loglevels detail logging debug will also print filename and line number debug logging debug info logging info warning logging warning error logging error critical logging critical defaultloglevel logging warning tqdmactive true def getdefaultlogginglevel envlevelstr os getenvtransformersverbosity none if envlevelstr if envlevelstr in loglevels return loglevelsenvlevelstr else logging getlogger warning funknown option transformersverbosityenvlevelstr fhas to be one of joinloglevels keys return defaultloglevel def getlibraryname str return name split 0 def getlibraryrootlogger logging logger return logging getloggergetlibraryname def configurelibraryrootlogger none global defaulthandler with lock if defaulthandler this library has already configured the library root logger return defaulthandler logging streamhandler set sys stderr as stream set defaults based on https github compyinstallerpyinstallerissues7334issuecomment1357447176 if sys stderr is none sys stderr openos devnull w defaulthandler flush sys stderr flush apply our default configuration to the library root logger libraryrootlogger getlibraryrootlogger libraryrootlogger addhandlerdefaulthandler libraryrootlogger setlevelgetdefaultlogginglevel if logging level is debug we add pathname and lineno to formatter for easy debugging if os getenvtransformersverbosity none detail formatter logging formatterlevelnamespathnames linenos asctimes messages defaulthandler setformatterformatter libraryrootlogger propagate false def resetlibraryrootlogger none global defaulthandler with lock if not defaulthandler return libraryrootlogger getlibraryrootlogger libraryrootlogger removehandlerdefaulthandler libraryrootlogger setlevellogging notset defaulthandler none def getloglevelsdict return loglevels def capturewarningscapture logger getloggerpy warnings if not logger handlers logger addhandlerdefaulthandler logger setlevelgetlibraryrootlogger level capturewarningscapture def getloggername optionalstr none logging logger if name is none name getlibraryname configurelibraryrootlogger return logging getloggername def getverbosity int set the verbosity level for the transformers s root logger args verbosity int logging level e g one of transformers logging critical or transformers logging fatal transformers logging error transformers logging warning or transformers logging warn transformers logging info transformers logging debug set the verbosity to the info level return setverbosityinfo def setverbositywarning set the verbosity to the debug level return setverbositydebug def setverbosityerror disable the default handler of the huggingface transformers s root logger configurelibraryrootlogger assert defaulthandler is not none getlibraryrootlogger removehandlerdefaulthandler def enabledefaulthandler none adds a handler to the huggingface transformers s root logger configurelibraryrootlogger assert handler is not none getlibraryrootlogger addhandlerhandler def removehandlerhandler logging handler none disable propagation of the library log outputs note that log propagation is disabled by default enable propagation of the library log outputs please disable the huggingface transformers s default handler to prevent double logging if the root logger has been configured enable explicit formatting for every huggingface transformers s logger the explicit formatter is as follows levelnamefilenameline number time message all handlers currently bound to the root logger are affected by this method resets the formatting for huggingface transformers s loggers all handlers currently bound to the root logger are affected by this method this method is identical to logger warning but if env var transformersnoadvisorywarnings1 is set this warning will not be printed this method is identical to logger warning but will emit the warning with the same message only once note the cache is for the function arguments so 2 different callers using the same arguments will hit the cache the assumption here is that all warning messages are unique across the code if they aren t then need to switch to another type of cache that includes the caller frame information in the hashing function dummy tqdm which doesn t do anything def initself args kwargs pylint disableunusedargument self iterator args0 if args else none def iterself return iterself iterator def getattrself return a boolean indicating whether tqdm progress bars are enabled global tqdmactive return booltqdmactive def enableprogressbar disable tqdm progress bar global tqdmactive tqdmactive false hfhubutils disableprogressbars coding utf 8 2020 optuna hugging face licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license logging utilities noqa noqa noqa noqa noqa noqa noqa noqa will also print filename and line number if transformers_verbosity env var is set to one of the valid choices return that as the new default level if it is not fall back to _default_log_level this library has already configured the library root logger set sys stderr as stream set defaults based on https github com pyinstaller pyinstaller issues 7334 issuecomment 1357447176 apply our default configuration to the library root logger if logging level is debug we add pathname and lineno to formatter for easy debugging calls the capturewarnings method from the logging library to enable management of the warnings emitted by the warnings library read more about this method here https docs python org 3 library logging html integration with the warnings module all warnings will be logged through the py warnings logger careful this method also adds a handler to this logger if it does not already have one and updates the logging level of that logger to the library s root logger return a logger with the specified name this function is not supposed to be directly accessed unless you are writing a custom transformers module return the current level for the transformers s root logger as an int returns int the logging level tip transformers has following logging levels 50 transformers logging critical or transformers logging fatal 40 transformers logging error 30 transformers logging warning or transformers logging warn 20 transformers logging info 10 transformers logging debug tip set the verbosity level for the transformers s root logger args verbosity int logging level e g one of transformers logging critical or transformers logging fatal transformers logging error transformers logging warning or transformers logging warn transformers logging info transformers logging debug set the verbosity to the info level set the verbosity to the warning level set the verbosity to the debug level set the verbosity to the error level disable the default handler of the huggingface transformers s root logger enable the default handler of the huggingface transformers s root logger adds a handler to the huggingface transformers s root logger removes given handler from the huggingface transformers s root logger disable propagation of the library log outputs note that log propagation is disabled by default enable propagation of the library log outputs please disable the huggingface transformers s default handler to prevent double logging if the root logger has been configured enable explicit formatting for every huggingface transformers s logger the explicit formatter is as follows levelname filename line number time message all handlers currently bound to the root logger are affected by this method resets the formatting for huggingface transformers s loggers all handlers currently bound to the root logger are affected by this method this method is identical to logger warning but if env var transformers_no_advisory_warnings 1 is set this warning will not be printed this method is identical to logger warning but will emit the warning with the same message only once note the cache is for the function arguments so 2 different callers using the same arguments will hit the cache the assumption here is that all warning messages are unique across the code if they aren t then need to switch to another type of cache that includes the caller frame information in the hashing function dummy tqdm which doesn t do anything pylint disable unused argument return empty function pylint disable unused argument return a boolean indicating whether tqdm progress bars are enabled enable tqdm progress bar disable tqdm progress bar
import functools
import logging
import os
import sys
import threading
from logging import (
    CRITICAL,  
    DEBUG,  
    ERROR,  
    FATAL,  
    INFO,  
    NOTSET,  
    WARN,  
    WARNING,  
)
from logging import captureWarnings as _captureWarnings
from typing import Optional

import huggingface_hub.utils as hf_hub_utils
from tqdm import auto as tqdm_lib


_lock = threading.Lock()
_default_handler: Optional[logging.Handler] = None

log_levels = {
    "detail": logging.DEBUG,  
    "debug": logging.DEBUG,
    "info": logging.INFO,
    "warning": logging.WARNING,
    "error": logging.ERROR,
    "critical": logging.CRITICAL,
}

_default_log_level = logging.WARNING

_tqdm_active = True


def _get_default_logging_level():
    
    env_level_str = os.getenv("TRANSFORMERS_VERBOSITY", None)
    if env_level_str:
        if env_level_str in log_levels:
            return log_levels[env_level_str]
        else:
            logging.getLogger().warning(
                f"Unknown option TRANSFORMERS_VERBOSITY={env_level_str}, "
                f"has to be one of: { ', '.join(log_levels.keys()) }"
            )
    return _default_log_level


def _get_library_name() -> str:
    return __name__.split(".")[0]


def _get_library_root_logger() -> logging.Logger:
    return logging.getLogger(_get_library_name())


def _configure_library_root_logger() -> None:
    global _default_handler

    with _lock:
        if _default_handler:
            
            return
        _default_handler = logging.StreamHandler()  
        
        if sys.stderr is None:
            sys.stderr = open(os.devnull, "w")

        _default_handler.flush = sys.stderr.flush

        
        library_root_logger = _get_library_root_logger()
        library_root_logger.addHandler(_default_handler)
        library_root_logger.setLevel(_get_default_logging_level())
        
        if os.getenv("TRANSFORMERS_VERBOSITY", None) == "detail":
            formatter = logging.Formatter("[%(levelname)s|%(pathname)s:%(lineno)s] %(asctime)s >> %(message)s")
            _default_handler.setFormatter(formatter)

        library_root_logger.propagate = False


def _reset_library_root_logger() -> None:
    global _default_handler

    with _lock:
        if not _default_handler:
            return

        library_root_logger = _get_library_root_logger()
        library_root_logger.removeHandler(_default_handler)
        library_root_logger.setLevel(logging.NOTSET)
        _default_handler = None


def get_log_levels_dict():
    return log_levels


def captureWarnings(capture):
    
    logger = get_logger("py.warnings")

    if not logger.handlers:
        logger.addHandler(_default_handler)

    logger.setLevel(_get_library_root_logger().level)

    _captureWarnings(capture)


def get_logger(name: Optional[str] = None) -> logging.Logger:
    

    if name is None:
        name = _get_library_name()

    _configure_library_root_logger()
    return logging.getLogger(name)


def get_verbosity() -> int:
    

    _configure_library_root_logger()
    return _get_library_root_logger().getEffectiveLevel()


def set_verbosity(verbosity: int) -> None:
    

    _configure_library_root_logger()
    _get_library_root_logger().setLevel(verbosity)


def set_verbosity_info():
    
    return set_verbosity(INFO)


def set_verbosity_warning():
    
    return set_verbosity(WARNING)


def set_verbosity_debug():
    
    return set_verbosity(DEBUG)


def set_verbosity_error():
    
    return set_verbosity(ERROR)


def disable_default_handler() -> None:
    

    _configure_library_root_logger()

    assert _default_handler is not None
    _get_library_root_logger().removeHandler(_default_handler)


def enable_default_handler() -> None:
    

    _configure_library_root_logger()

    assert _default_handler is not None
    _get_library_root_logger().addHandler(_default_handler)


def add_handler(handler: logging.Handler) -> None:
    

    _configure_library_root_logger()

    assert handler is not None
    _get_library_root_logger().addHandler(handler)


def remove_handler(handler: logging.Handler) -> None:
    

    _configure_library_root_logger()

    assert handler is not None and handler not in _get_library_root_logger().handlers
    _get_library_root_logger().removeHandler(handler)


def disable_propagation() -> None:
    

    _configure_library_root_logger()
    _get_library_root_logger().propagate = False


def enable_propagation() -> None:
    

    _configure_library_root_logger()
    _get_library_root_logger().propagate = True


def enable_explicit_format() -> None:
    
    handlers = _get_library_root_logger().handlers

    for handler in handlers:
        formatter = logging.Formatter("[%(levelname)s|%(filename)s:%(lineno)s] %(asctime)s >> %(message)s")
        handler.setFormatter(formatter)


def reset_format() -> None:
    
    handlers = _get_library_root_logger().handlers

    for handler in handlers:
        handler.setFormatter(None)


def warning_advice(self, *args, **kwargs):
    
    no_advisory_warnings = os.getenv("TRANSFORMERS_NO_ADVISORY_WARNINGS", False)
    if no_advisory_warnings:
        return
    self.warning(*args, **kwargs)


logging.Logger.warning_advice = warning_advice


@functools.lru_cache(None)
def warning_once(self, *args, **kwargs):
    
    self.warning(*args, **kwargs)


logging.Logger.warning_once = warning_once


class EmptyTqdm:
    

    def __init__(self, *args, **kwargs):  
        self._iterator = args[0] if args else None

    def __iter__(self):
        return iter(self._iterator)

    def __getattr__(self, _):
        

        def empty_fn(*args, **kwargs):  
            return

        return empty_fn

    def __enter__(self):
        return self

    def __exit__(self, type_, value, traceback):
        return


class _tqdm_cls:
    def __call__(self, *args, **kwargs):
        if _tqdm_active:
            return tqdm_lib.tqdm(*args, **kwargs)
        else:
            return EmptyTqdm(*args, **kwargs)

    def set_lock(self, *args, **kwargs):
        self._lock = None
        if _tqdm_active:
            return tqdm_lib.tqdm.set_lock(*args, **kwargs)

    def get_lock(self):
        if _tqdm_active:
            return tqdm_lib.tqdm.get_lock()


tqdm = _tqdm_cls()


def is_progress_bar_enabled() -> bool:
    
    global _tqdm_active
    return bool(_tqdm_active)


def enable_progress_bar():
    
    global _tqdm_active
    _tqdm_active = True
    hf_hub_utils.enable_progress_bars()


def disable_progress_bar():
    
    global _tqdm_active
    _tqdm_active = False
    hf_hub_utils.disable_progress_bars()