<commit_before>#!/usr/bin/env python3
# coding=utf-8
#
# Top-level docs builder
#
# This is just a front-end to sphinx-build that can call it multiple times for different language/target combinations
#
# Will build out to _build/LANG/TARGET by default
#
# Specific custom docs functionality should be added in conf_common.py or in a Sphinx extension, not here.
#
from __future__ import print_function
import argparse
import math
import multiprocessing
import os
import os.path
import subprocess
import sys
import re
from collections import namedtuple

LANGUAGES = ["en", "zh_CN"]
TARGETS = ["esp32", "esp32s2"]

SPHINX_WARN_LOG = "sphinx-warning-log.txt"
SPHINX_SANITIZED_LOG = "sphinx-warning-log-sanitized.txt"
SPHINX_KNOWN_WARNINGS = os.path.join(os.environ["IDF_PATH"], "docs", "sphinx-known-warnings.txt")

DXG_WARN_LOG = "doxygen-warning-log.txt"
DXG_SANITIZED_LOG = "doxygen-warning-log-sanitized.txt"
DXG_KNOWN_WARNINGS = os.path.join(os.environ["IDF_PATH"], "docs", "doxygen-known-warnings.txt")

LogMessage = namedtuple("LogMessage", "original_text sanitized_text")

languages = LANGUAGES
targets = TARGETS


def main():
    # check Python dependencies for docs
    try:
        subprocess.check_call([sys.executable,
                               os.path.join(os.environ["IDF_PATH"],
                                            "tools",
                                            "check_python_dependencies.py"),
                               "-r",
                               "{}/docs/requirements.txt".format(os.environ["IDF_PATH"])
                               ])
    except subprocess.CalledProcessError:
        raise SystemExit(2)  # stdout will already have these errors<commit_msg>docs: Require UTF-8 as default encoding to build docs<commit_after>#!/usr/bin/env python3
# coding=utf-8
#
# Top-level docs builder
#
# This is just a front-end to sphinx-build that can call it multiple times for different language/target combinations
#
# Will build out to _build/LANG/TARGET by default
#
# Specific custom docs functionality should be added in conf_common.py or in a Sphinx extension, not here.
#
from __future__ import print_function
import argparse
import locale
import math
import multiprocessing
import os
import os.path
import subprocess
import sys
import re
from collections import namedtuple

LANGUAGES = ["en", "zh_CN"]
TARGETS = ["esp32", "esp32s2"]

SPHINX_WARN_LOG = "sphinx-warning-log.txt"
SPHINX_SANITIZED_LOG = "sphinx-warning-log-sanitized.txt"
SPHINX_KNOWN_WARNINGS = os.path.join(os.environ["IDF_PATH"], "docs", "sphinx-known-warnings.txt")

DXG_WARN_LOG = "doxygen-warning-log.txt"
DXG_SANITIZED_LOG = "doxygen-warning-log-sanitized.txt"
DXG_KNOWN_WARNINGS = os.path.join(os.environ["IDF_PATH"], "docs", "doxygen-known-warnings.txt")

LogMessage = namedtuple("LogMessage", "original_text sanitized_text")

languages = LANGUAGES
targets = TARGETS


def main():
    # check Python dependencies for docs
    try:
        subprocess.check_call([sys.executable,
                               os.path.join(os.environ["IDF_PATH"],
                                            "tools",
                                            "check_python_dependencies.py"),
                               "-r",
                               "{}/docs/requirements.txt".format(os.environ["IDF_PATH"])
                               ])
    except subprocess.CalledProcessError:
        raise SystemExit(2)  # stdout will already have these errors

    # This is not the only way to make sure that all files opened by Python are treated as UTF-8, but the other way is passing encoding='utf-8' to all open()
    # functions and this way makes Python 2 compatibility really tough if there is any code that assumes text files contain strings (kconfiglib assumes this).
    # The reason for that is that you need to import io.open() to support the encoding argument on Python 2, and this function always uses Py2's unicode
    # type not the str type.
    if 'UTF-8' not in locale.getlocale():
        raise RuntimeError("build_docs.py requires the default locale's encoding to be UTF-8. " +
                           "Setting environment variable LC_ALL=C.UTF-8 when running build_docs.py may be enough to fix this.")
{'fraction_non_alphanumeric': 0.0924964605946201, 'fraction_numerical': 0.005899008966493629, 'mean_word_length': 2.8747714808043874, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6111555', 'n_tokens_mistral': 1285, 'n_tokens_neox': 1240, 'n_words': 369}