<commit_before>import dbt.flags
import dbt.ui.colors

import json
import logging
import os
import sys
import warnings
from dataclasses import dataclass
from datetime import datetime
from typing import Optional, List, ContextManager, Callable, Dict, Any, Set

import colorama
import logbook
from hologram import JsonSchemaMixin

# Colorama needs some help on windows because we're using logger.info
# intead of print(). If the Windows env doesn't have a TERM var set,
# then we should override the logging stream to use the colorama
# converter. If the TERM var is set (as with Git Bash), then it's safe
# to send escape characters and no log handler injection is needed.
colorama_stdout = sys.stdout
colorama_wrap = True

colorama.init(wrap=colorama_wrap)


if sys.platform == 'win32' and not os.getenv('TERM'):
    colorama_wrap = False
    colorama_stdout = colorama.AnsiToWin32(sys.stdout).stream

elif sys.platform == 'win32':
    colorama_wrap = False

colorama.init(wrap=colorama_wrap)


STDOUT_LOG_FORMAT = '{record.message}'
# TODO: can we change the time to just "{record.time:%Y-%m-%d %H:%M:%S.%f%z}"?
DEBUG_LOG_FORMAT = (
    '{record.time:%Y-%m-%d %H:%M:%S%z},{record.time.microsecond:03} '
    '({record.thread_name}): '
    '{record.message}'
)


ExceptionInformation = str
Extras = Dict[str, Any]


@dataclass
class LogMessage(JsonSchemaMixin):
    timestamp: datetime
    message: str
    channel: str
    level: int
    levelname: str
    thread_name: str
    process: int
    extra: Optional[Extras] = None
    exc_info: Optional[ExceptionInformation] = None

    @classmethod
    def from_record_formatted(cls, record: logbook.LogRecord, message: str):
        extra = dict(record.extra)
        log_message = LogMessage(
            timestamp=record.time,
            message=message,
            channel=record.channel,
            level=record.level,
            levelname=logbook.get_level_name(record.level),
            extra=extra,
            thread_name=record.thread_name,
            process=record.process,
            exc_info=record.formatted_exception,
        )
        return log_message


class LogMessageFormatter(logbook.StringFormatter):
    def __call__(self, record, handler):
        data = self.format_record(record, handler)
        exc = self.format_exception(record)
        if exc:
            data.exc_info = exc
        return data

    def format_record(self, record, handler):
        message = super().format_record(record, handler)
        return LogMessage.from_record_formatted(record, message)


class JsonFormatter(LogMessageFormatter):
    def __call__(self, record, handler):
        """Return a the record converted to LogMessage's JSON form"""
        # utils imports exceptions which imports logger...
        import dbt.utils
        log_message = super().__call__(record, handler)
        return json.dumps(log_message.to_dict(), cls=dbt.utils.JSONEncoder)


class FormatterMixin:
    def __init__(self, format_string):
        self._text_format_string = format_string
        self.formatter_class = logbook.StringFormatter
        # triggers a formatter update via logbook.StreamHandler
        self.format_string = self._text_format_string

    def format_json(self):
        # set our formatter to the json formatter
        self.formatter_class = JsonFormatter
        self.format_string = STDOUT_LOG_FORMAT

    def format_text(self):
        # set our formatter to the regular stdout/stderr handler
        self.formatter_class = logbook.StringFormatter
        self.format_string = self._text_format_string


class OutputHandler(logbook.StreamHandler, FormatterMixin):
    """Output handler.

    The `format_string` parameter only changes the default text output, not
      debug mode or json.
    """
    def __init__(
        self,
        stream,
        level=logbook.INFO,
        format_string=STDOUT_LOG_FORMAT,
        bubble=True,
    ) -> None:
        self._default_format = format_string
        logbook.StreamHandler.__init__(
            self,
            stream=stream,
            level=level,
            format_string=format_string,
            bubble=bubble,
        )
        FormatterMixin.__init__(self, format_string)

    def set_text_format(self, format_string: str):
        """Set the text format to format_string. In JSON output mode, this is
        a noop.
        """
        if self.formatter_class is logbook.StringFormatter:
            # reset text format
            self._text_format_string = format_string
            self.format_text()

    def reset(self):
        self.level = logbook.INFO
        self._text_format_string = self._default_format
        self.format_text()

    def should_handle(self, record):
        if record.level < self.level:
            return False
        text_mode = self.formatter_class is logbook.StringFormatter
        if text_mode and record.extra.get('json_only', False):
            return False
        elif not text_mode and record.extra.get('text_only', False):
            return False
        else:
            return True


def _redirect_std_logging():
    logbook.compat.redirect_logging()


def _root_channel(record: logbook.LogRecord) -> str:
    return record.channel.split('.')[0]


class Relevel(logbook.Processor):
    def __init__(
        self,
        allowed: List[str],
        min_level=logbook.WARNING,
        target_level=logbook.DEBUG,
    ) -> None:
        self.allowed: Set[str] = set(allowed)
        self.min_level = min_level
        self.target_level = target_level
        super().__init__()

    def process(self, record):
        if _root_channel(record) in self.allowed:
            return
        record.extra['old_level'] = record.level
        # suppress logs at/below our min level by lowering them to NOTSET
        if record.level < self.min_level:
            record.level = logbook.NOTSET
        # if we didn't mess with it, then lower all logs above our level to
        # our target level.
        else:
            record.level = self.target_level


class JsonOnly(logbook.Processor):
    def process(self, record):
        record.extra['json_only'] = True


class TextOnly(logbook.Processor):
    def process(self, record):
        record.extra['text_only'] = True


class TimingProcessor(logbook.Processor):
    def __init__(self, timing_info: Optional[JsonSchemaMixin] = None):
        self.timing_info = timing_info
        super().__init__()

    def process(self, record):
        if self.timing_info is not None:
            record.extra['timing_info'] = self.timing_info.to_dict()


class DbtProcessState(logbook.Processor):
    def __init__(self, value: str):
        self.value = value
        super().__init__()

    def process(self, record):
        overwrite = (
            'run_state' not in record.extra or
            record.extra['run_state'] == 'internal'
        )
        if overwrite:
            record.extra['run_state'] = self.value


class DbtModelState(logbook.Processor):
    def __init__(self, state: Dict[str, str]):
        self.state = state
        super().__init__()

    def process(self, record):
        record.extra.update(self.state)


class DbtStatusMessage(logbook.Processor):
    def process(self, record):
        record.extra['is_status_message'] = True


class UniqueID(logbook.Processor):
    def __init__(self, unique_id: str):
        self.unique_id = unique_id
        super().__init__()

    def process(self, record):
        record.extra['unique_id'] = self.unique_id


class NodeCount(logbook.Processor):
    def __init__(self, node_count: int):
        self.node_count = node_count
        super().__init__()

    def process(self, record):
        record.extra['node_count'] = self.node_count


class NodeMetadata(logbook.Processor):
    def __init__(self, node, node_index):
        self.node = node
        self.node_index = node_index
        super().__init__()

    def process(self, record):
        keys = [
            ('alias', 'node_alias'),
            ('schema', 'node_schema'),
            ('database', 'node_database'),
            ('name', 'node_name'),
            ('original_file_path', 'node_path'),
            ('resource_type', 'resource_type'),
        ]
        for attr, key in keys:
            value = getattr(self.node, attr, None)
            if value is not None:
                record.extra[key] = value
        record.extra['node_index'] = self.node_index
        if hasattr(self.node, 'config'):
            materialized = getattr(self.node.config, 'materialized', None)
            if materialized is not None:
                record.extra['node_materialized'] = materialized


class TimestampNamed(JsonOnly):
    def __init__(self, name: str):
        self.name = name
        super().__init__()

    def process(self, record):
        super().process(record)
        record.extra[self.name] = datetime.utcnow().isoformat()


logger = logbook.Logger('dbt')
# provide this for the cache, disabled by default
CACHE_LOGGER = logbook.Logger('dbt.cache')
CACHE_LOGGER.disable()

warnings.filterwarnings("ignore", category=ResourceWarning,
                        message="unclosed.*<socket.socket.*>")

initialized = False


def make_log_dir_if_missing(log_dir):
    import dbt.clients.system
    dbt.clients.system.make_directory(log_dir)


class DebugWarnings(logbook.compat.redirected_warnings):
    """Log warnings, except send them to 'debug' instead of 'warning' level.
    """
    def make_record(self, message, exception, filename, lineno):
        rv = super().make_record(message, exception, filename, lineno)
        rv.level = logbook.DEBUG
        rv.extra['from_warnings'] = True
        return rv


# push Python warnings to debug level logs. This will suppress all import-time
# warnings.
DebugWarnings().__enter__()
# redirect stdlib logging to logbook
_redirect_std_logging()


class DelayedFileHandler(logbook.TimedRotatingFileHandler, FormatterMixin):
    def __init__(
        self,
        log_dir: Optional[str] = None,
        level=logbook.DEBUG,
        filter=None,
        bubble=True,
    ) -> None:
        self.disabled = False
        self._msg_buffer: Optional[List[logbook.LogRecord]] = []
        # if we get 1k messages without a logfile being set, something is wrong
        self._bufmax = 1000
        self._log_path = None
        # we need the base handler class' __init__ to run so handling works
        logbook.Handler.__init__(self, level, filter, bubble)
        if log_dir is not None:
            self.set_path(log_dir)
        self._text_format_string = None

    def reset(self):
        if self.initialized:
            self.close()
        self._log_path = None
        self._msg_buffer = []
        self.disabled = False

    @property
    def initialized(self):
        return self._log_path is not None

    def set_path(self, log_dir):
        """log_dir can be the path to a log directory, or `None` to avoid
        writing to a file (for `dbt debug`).
        """
        if self.disabled:
            return

        assert not self.initialized, 'set_path called after being set'

        if log_dir is None:
            self.disabled = True
            return

        make_log_dir_if_missing(log_dir)
        log_path = os.path.join(log_dir, 'dbt.log')
        self._super_init(log_path)
        self._replay_buffered()
        self._log_path = log_path

    def _super_init(self, log_path):
        logbook.TimedRotatingFileHandler.__init__(
            self,
            filename=log_path,
            level=self.level,
            filter=self.filter,
            bubble=self.bubble,
            format_string=DEBUG_LOG_FORMAT,
            date_format='%Y-%m-%d',
            backup_count=7,
            timed_filename_for_current=False,
        )
        FormatterMixin.__init__(self, DEBUG_LOG_FORMAT)

    def _replay_buffered(self):
        for record in self._msg_buffer:<commit_msg>rotate the file at 10mb instead of by date<commit_after>import dbt.flags
import dbt.ui.colors

import json
import logging
import os
import sys
import warnings
from dataclasses import dataclass
from datetime import datetime
from typing import Optional, List, ContextManager, Callable, Dict, Any, Set

import colorama
import logbook
from hologram import JsonSchemaMixin

# Colorama needs some help on windows because we're using logger.info
# intead of print(). If the Windows env doesn't have a TERM var set,
# then we should override the logging stream to use the colorama
# converter. If the TERM var is set (as with Git Bash), then it's safe
# to send escape characters and no log handler injection is needed.
colorama_stdout = sys.stdout
colorama_wrap = True

colorama.init(wrap=colorama_wrap)


if sys.platform == 'win32' and not os.getenv('TERM'):
    colorama_wrap = False
    colorama_stdout = colorama.AnsiToWin32(sys.stdout).stream

elif sys.platform == 'win32':
    colorama_wrap = False

colorama.init(wrap=colorama_wrap)


STDOUT_LOG_FORMAT = '{record.message}'
# TODO: can we change the time to just "{record.time:%Y-%m-%d %H:%M:%S.%f%z}"?
DEBUG_LOG_FORMAT = (
    '{record.time:%Y-%m-%d %H:%M:%S%z},{record.time.microsecond:03} '
    '({record.thread_name}): '
    '{record.message}'
)


ExceptionInformation = str
Extras = Dict[str, Any]


@dataclass
class LogMessage(JsonSchemaMixin):
    timestamp: datetime
    message: str
    channel: str
    level: int
    levelname: str
    thread_name: str
    process: int
    extra: Optional[Extras] = None
    exc_info: Optional[ExceptionInformation] = None

    @classmethod
    def from_record_formatted(cls, record: logbook.LogRecord, message: str):
        extra = dict(record.extra)
        log_message = LogMessage(
            timestamp=record.time,
            message=message,
            channel=record.channel,
            level=record.level,
            levelname=logbook.get_level_name(record.level),
            extra=extra,
            thread_name=record.thread_name,
            process=record.process,
            exc_info=record.formatted_exception,
        )
        return log_message


class LogMessageFormatter(logbook.StringFormatter):
    def __call__(self, record, handler):
        data = self.format_record(record, handler)
        exc = self.format_exception(record)
        if exc:
            data.exc_info = exc
        return data

    def format_record(self, record, handler):
        message = super().format_record(record, handler)
        return LogMessage.from_record_formatted(record, message)


class JsonFormatter(LogMessageFormatter):
    def __call__(self, record, handler):
        """Return a the record converted to LogMessage's JSON form"""
        # utils imports exceptions which imports logger...
        import dbt.utils
        log_message = super().__call__(record, handler)
        return json.dumps(log_message.to_dict(), cls=dbt.utils.JSONEncoder)


class FormatterMixin:
    def __init__(self, format_string):
        self._text_format_string = format_string
        self.formatter_class = logbook.StringFormatter
        # triggers a formatter update via logbook.StreamHandler
        self.format_string = self._text_format_string

    def format_json(self):
        # set our formatter to the json formatter
        self.formatter_class = JsonFormatter
        self.format_string = STDOUT_LOG_FORMAT

    def format_text(self):
        # set our formatter to the regular stdout/stderr handler
        self.formatter_class = logbook.StringFormatter
        self.format_string = self._text_format_string


class OutputHandler(logbook.StreamHandler, FormatterMixin):
    """Output handler.

    The `format_string` parameter only changes the default text output, not
      debug mode or json.
    """
    def __init__(
        self,
        stream,
        level=logbook.INFO,
        format_string=STDOUT_LOG_FORMAT,
        bubble=True,
    ) -> None:
        self._default_format = format_string
        logbook.StreamHandler.__init__(
            self,
            stream=stream,
            level=level,
            format_string=format_string,
            bubble=bubble,
        )
        FormatterMixin.__init__(self, format_string)

    def set_text_format(self, format_string: str):
        """Set the text format to format_string. In JSON output mode, this is
        a noop.
        """
        if self.formatter_class is logbook.StringFormatter:
            # reset text format
            self._text_format_string = format_string
            self.format_text()

    def reset(self):
        self.level = logbook.INFO
        self._text_format_string = self._default_format
        self.format_text()

    def should_handle(self, record):
        if record.level < self.level:
            return False
        text_mode = self.formatter_class is logbook.StringFormatter
        if text_mode and record.extra.get('json_only', False):
            return False
        elif not text_mode and record.extra.get('text_only', False):
            return False
        else:
            return True


def _redirect_std_logging():
    logbook.compat.redirect_logging()


def _root_channel(record: logbook.LogRecord) -> str:
    return record.channel.split('.')[0]


class Relevel(logbook.Processor):
    def __init__(
        self,
        allowed: List[str],
        min_level=logbook.WARNING,
        target_level=logbook.DEBUG,
    ) -> None:
        self.allowed: Set[str] = set(allowed)
        self.min_level = min_level
        self.target_level = target_level
        super().__init__()

    def process(self, record):
        if _root_channel(record) in self.allowed:
            return
        record.extra['old_level'] = record.level
        # suppress logs at/below our min level by lowering them to NOTSET
        if record.level < self.min_level:
            record.level = logbook.NOTSET
        # if we didn't mess with it, then lower all logs above our level to
        # our target level.
        else:
            record.level = self.target_level


class JsonOnly(logbook.Processor):
    def process(self, record):
        record.extra['json_only'] = True


class TextOnly(logbook.Processor):
    def process(self, record):
        record.extra['text_only'] = True


class TimingProcessor(logbook.Processor):
    def __init__(self, timing_info: Optional[JsonSchemaMixin] = None):
        self.timing_info = timing_info
        super().__init__()

    def process(self, record):
        if self.timing_info is not None:
            record.extra['timing_info'] = self.timing_info.to_dict()


class DbtProcessState(logbook.Processor):
    def __init__(self, value: str):
        self.value = value
        super().__init__()

    def process(self, record):
        overwrite = (
            'run_state' not in record.extra or
            record.extra['run_state'] == 'internal'
        )
        if overwrite:
            record.extra['run_state'] = self.value


class DbtModelState(logbook.Processor):
    def __init__(self, state: Dict[str, str]):
        self.state = state
        super().__init__()

    def process(self, record):
        record.extra.update(self.state)


class DbtStatusMessage(logbook.Processor):
    def process(self, record):
        record.extra['is_status_message'] = True


class UniqueID(logbook.Processor):
    def __init__(self, unique_id: str):
        self.unique_id = unique_id
        super().__init__()

    def process(self, record):
        record.extra['unique_id'] = self.unique_id


class NodeCount(logbook.Processor):
    def __init__(self, node_count: int):
        self.node_count = node_count
        super().__init__()

    def process(self, record):
        record.extra['node_count'] = self.node_count


class NodeMetadata(logbook.Processor):
    def __init__(self, node, node_index):
        self.node = node
        self.node_index = node_index
        super().__init__()

    def process(self, record):
        keys = [
            ('alias', 'node_alias'),
            ('schema', 'node_schema'),
            ('database', 'node_database'),
            ('name', 'node_name'),
            ('original_file_path', 'node_path'),
            ('resource_type', 'resource_type'),
        ]
        for attr, key in keys:
            value = getattr(self.node, attr, None)
            if value is not None:
                record.extra[key] = value
        record.extra['node_index'] = self.node_index
        if hasattr(self.node, 'config'):
            materialized = getattr(self.node.config, 'materialized', None)
            if materialized is not None:
                record.extra['node_materialized'] = materialized


class TimestampNamed(JsonOnly):
    def __init__(self, name: str):
        self.name = name
        super().__init__()

    def process(self, record):
        super().process(record)
        record.extra[self.name] = datetime.utcnow().isoformat()


logger = logbook.Logger('dbt')
# provide this for the cache, disabled by default
CACHE_LOGGER = logbook.Logger('dbt.cache')
CACHE_LOGGER.disable()

warnings.filterwarnings("ignore", category=ResourceWarning,
                        message="unclosed.*<socket.socket.*>")

initialized = False


def make_log_dir_if_missing(log_dir):
    import dbt.clients.system
    dbt.clients.system.make_directory(log_dir)


class DebugWarnings(logbook.compat.redirected_warnings):
    """Log warnings, except send them to 'debug' instead of 'warning' level.
    """
    def make_record(self, message, exception, filename, lineno):
        rv = super().make_record(message, exception, filename, lineno)
        rv.level = logbook.DEBUG
        rv.extra['from_warnings'] = True
        return rv


# push Python warnings to debug level logs. This will suppress all import-time
# warnings.
DebugWarnings().__enter__()
# redirect stdlib logging to logbook
_redirect_std_logging()


class DelayedFileHandler(logbook.RotatingFileHandler, FormatterMixin):
    def __init__(
        self,
        log_dir: Optional[str] = None,
        level=logbook.DEBUG,
        filter=None,
        bubble=True,
        max_size=10 * 1024 * 1024,  # 10 mb
        backup_count=5,
    ) -> None:
        self.disabled = False
        self._msg_buffer: Optional[List[logbook.LogRecord]] = []
        # if we get 1k messages without a logfile being set, something is wrong
        self._bufmax = 1000
        self._log_path = None
        # we need the base handler class' __init__ to run so handling works
        logbook.Handler.__init__(self, level, filter, bubble)
        if log_dir is not None:
            self.set_path(log_dir)
        self._text_format_string = None
        self._max_size = max_size
        self._backup_count = backup_count

    def reset(self):
        if self.initialized:
            self.close()
        self._log_path = None
        self._msg_buffer = []
        self.disabled = False

    @property
    def initialized(self):
        return self._log_path is not None

    def set_path(self, log_dir):
        """log_dir can be the path to a log directory, or `None` to avoid
        writing to a file (for `dbt debug`).
        """
        if self.disabled:
            return

        assert not self.initialized, 'set_path called after being set'

        if log_dir is None:
            self.disabled = True
            return

        make_log_dir_if_missing(log_dir)
        log_path = os.path.join(log_dir, 'dbt.log')
        self._super_init(log_path)
        self._replay_buffered()
        self._log_path = log_path

    def _super_init(self, log_path):
        logbook.RotatingFileHandler.__init__(
            self,
            filename=log_path,
            level=self.level,
            filter=self.filter,
            delay=True,
            max_size=self._max_size,
            backup_count=self._backup_count,
            bubble=self.bubble,
            format_string=DEBUG_LOG_FORMAT,
        )
        FormatterMixin.__init__(self, DEBUG_LOG_FORMAT)

    def _replay_buffered(self):
        for record in self._msg_buffer:
{'fraction_non_alphanumeric': 0.09476620726985052, 'fraction_numerical': 0.0018320356414206604, 'mean_word_length': 2.5874533233756534, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3468039', 'n_tokens_mistral': 7017, 'n_tokens_neox': 6845, 'n_words': 1741}