<commit_before># -*- coding: utf-8 -*-

# Copyright © 2012-2019 <NAME> and others.

# Permission is hereby granted, free of charge, to any
# person obtaining a copy of this software and associated
# documentation files (the "Software"), to deal in the
# Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the
# Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice
# shall be included in all copies or substantial portions of
# the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

"""Utility functions."""

import configparser
import datetime
import hashlib
import io
import operator
import os
import re
import json
import shutil
import socket
import subprocess
import sys
import threading
import typing
from collections import defaultdict, OrderedDict
from collections.abc import Callable, Iterable
from html import unescape as html_unescape
from importlib import reload as _reload
from unicodedata import normalize as unicodenormalize
from urllib.parse import quote as urlquote
from urllib.parse import unquote as urlunquote
from urllib.parse import urlparse, urlunparse
from zipfile import ZipFile as zipf

import babel.dates
import dateutil.parser
import dateutil.tz
import PyRSS2Gen as rss
from blinker import signal
from doit import tools
from doit.cmdparse import CmdParse
from pkg_resources import resource_filename
from pygments.formatters import HtmlFormatter
from unidecode import unidecode

# Renames
from nikola import DEBUG  # NOQA
from .log import LOGGER, get_logger  # NOQA
from .hierarchy_utils import TreeNode, clone_treenode, flatten_tree_structure, sort_classifications
from .hierarchy_utils import join_hierarchical_category_path, parse_escaped_hierarchical_category_name

try:
    import toml
except ImportError:
    toml = None

try:
    from ruamel.yaml import YAML
except ImportError:
    YAML = None

try:
    import husl
except ImportError:
    husl = None

__all__ = ('CustomEncoder', 'get_theme_path', 'get_theme_path_real',
           'get_theme_chain', 'load_messages', 'copy_tree', 'copy_file',
           'slugify', 'unslugify', 'to_datetime', 'apply_filters',
           'config_changed', 'get_crumbs', 'get_tzname', 'get_asset_path',
           '_reload', 'unicode_str', 'bytes_str', 'unichr', 'Functionary',
           'TranslatableSetting', 'TemplateHookRegistry', 'LocaleBorg',
           'sys_encode', 'sys_decode', 'makedirs', 'get_parent_theme_name',
           'demote_headers', 'get_translation_candidate', 'write_metadata',
           'ask', 'ask_yesno', 'options2docstring', 'os_path_split',
           'get_displayed_page_number', 'adjust_name_for_index_path_list',
           'adjust_name_for_index_path', 'adjust_name_for_index_link',
           'NikolaPygmentsHTML', 'create_redirect', 'clean_before_deployment',
           'sort_posts', 'smartjoin', 'indent', 'load_data', 'html_unescape',
           'rss_writer', 'map_metadata', 'req_missing',
           # Deprecated, moved to hierarchy_utils:
           'TreeNode', 'clone_treenode', 'flatten_tree_structure',
           'sort_classifications', 'join_hierarchical_category_path',
           'parse_escaped_hierarchical_category_name',)

# Are you looking for 'generic_rss_renderer'?
# It's defined in nikola.nikola.Nikola (the site object).

# Aliases, previously for Python 2/3 compatibility.
bytes_str = bytes
unicode_str = str
unichr = chr


USE_SLUGIFY = True


def req_missing(names, purpose, python=True, optional=False):
    """Log that we are missing some requirements.

    `names` is a list/tuple/set of missing things.
    `purpose` is a string, specifying the use of the missing things.
              It completes the sentence:
                  In order to {purpose}, you must install ...
    `python` specifies whether the requirements are Python packages
                               or other software.
    `optional` specifies whether the things are required
                                 (this is an error and we exit with code 5)<commit_msg>Add dummy STDERR_HANDLER to support older plugins<commit_after># -*- coding: utf-8 -*-

# Copyright © 2012-2019 <NAME> and others.

# Permission is hereby granted, free of charge, to any
# person obtaining a copy of this software and associated
# documentation files (the "Software"), to deal in the
# Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the
# Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice
# shall be included in all copies or substantial portions of
# the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
# OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

"""Utility functions."""

import configparser
import datetime
import hashlib
import io
import operator
import os
import re
import json
import shutil
import socket
import subprocess
import sys
import threading
import typing
from collections import defaultdict, OrderedDict
from collections.abc import Callable, Iterable
from html import unescape as html_unescape
from importlib import reload as _reload
from unicodedata import normalize as unicodenormalize
from urllib.parse import quote as urlquote
from urllib.parse import unquote as urlunquote
from urllib.parse import urlparse, urlunparse
from zipfile import ZipFile as zipf

import babel.dates
import dateutil.parser
import dateutil.tz
import PyRSS2Gen as rss
from blinker import signal
from doit import tools
from doit.cmdparse import CmdParse
from pkg_resources import resource_filename
from pygments.formatters import HtmlFormatter
from unidecode import unidecode

# Renames
from nikola import DEBUG  # NOQA
from .log import LOGGER, get_logger  # NOQA
from .hierarchy_utils import TreeNode, clone_treenode, flatten_tree_structure, sort_classifications
from .hierarchy_utils import join_hierarchical_category_path, parse_escaped_hierarchical_category_name

try:
    import toml
except ImportError:
    toml = None

try:
    from ruamel.yaml import YAML
except ImportError:
    YAML = None

try:
    import husl
except ImportError:
    husl = None

__all__ = ('CustomEncoder', 'get_theme_path', 'get_theme_path_real',
           'get_theme_chain', 'load_messages', 'copy_tree', 'copy_file',
           'slugify', 'unslugify', 'to_datetime', 'apply_filters',
           'config_changed', 'get_crumbs', 'get_tzname', 'get_asset_path',
           '_reload', 'unicode_str', 'bytes_str', 'unichr', 'Functionary',
           'TranslatableSetting', 'TemplateHookRegistry', 'LocaleBorg',
           'sys_encode', 'sys_decode', 'makedirs', 'get_parent_theme_name',
           'demote_headers', 'get_translation_candidate', 'write_metadata',
           'ask', 'ask_yesno', 'options2docstring', 'os_path_split',
           'get_displayed_page_number', 'adjust_name_for_index_path_list',
           'adjust_name_for_index_path', 'adjust_name_for_index_link',
           'NikolaPygmentsHTML', 'create_redirect', 'clean_before_deployment',
           'sort_posts', 'smartjoin', 'indent', 'load_data', 'html_unescape',
           'rss_writer', 'map_metadata', 'req_missing',
           # Deprecated, moved to hierarchy_utils:
           'TreeNode', 'clone_treenode', 'flatten_tree_structure',
           'sort_classifications', 'join_hierarchical_category_path',
           'parse_escaped_hierarchical_category_name',)

# Are you looking for 'generic_rss_renderer'?
# It's defined in nikola.nikola.Nikola (the site object).

# Aliases, previously for Python 2/3 compatibility.
bytes_str = bytes
unicode_str = str
unichr = chr

# For compatibility with old logging setups.
# TODO remove in v9?
STDERR_HANDLER = None


USE_SLUGIFY = True


def req_missing(names, purpose, python=True, optional=False):
    """Log that we are missing some requirements.

    `names` is a list/tuple/set of missing things.
    `purpose` is a string, specifying the use of the missing things.
              It completes the sentence:
                  In order to {purpose}, you must install ...
    `python` specifies whether the requirements are Python packages
                               or other software.
    `optional` specifies whether the things are required
                                 (this is an error and we exit with code 5)
{'fraction_non_alphanumeric': 0.07227248536455562, 'fraction_numerical': 0.0030867482703565725, 'mean_word_length': 4.084415584415584, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2172959', 'n_tokens_mistral': 2682, 'n_tokens_neox': 2589, 'n_words': 993}