<commit_before>import os
import logging
from os.path import dirname
import platform
import sys

logger = logging.getLogger(__name__)

# MUST add pickle5 to the import path because it will be imported by some
# raylet modules.

if "pickle5" in sys.modules:
    import pkg_resources
    version_info = pkg_resources.require("pickle5")
    version = tuple(int(n) for n in version_info[0].version.split("."))
    if version < (0, 0, 10):
        raise ImportError("You are using an old version of pickle5 that "
                          "leaks memory, please run 'pip install pickle5 -U' "
                          "to upgrade")

if "OMP_NUM_THREADS" not in os.environ:
    logger.debug("[ray] Forcing OMP_NUM_THREADS=1 to avoid performance "
                 "degradation with many workers (issue #6998). You can "
                 "override this by explicitly setting OMP_NUM_THREADS.")
    os.environ["OMP_NUM_THREADS"] = "1"

# Add the directory containing pickle5 to the Python path so that we find the
# pickle5 version packaged with ray and not a pre-existing pickle5.
pickle5_path = os.path.join(
    os.path.abspath(os.path.dirname(__file__)), "pickle5_files")
sys.path.insert(0, pickle5_path)

# Importing psutil & setproctitle. Must be before ray._raylet is initialized.
thirdparty_files = os.path.join(
    os.path.abspath(os.path.dirname(__file__)), "thirdparty_files")<commit_msg>[Core] Warn when failed to get the exact version of pickle5 (#10731)<commit_after>import os
import logging
from os.path import dirname
import platform
import sys

logger = logging.getLogger(__name__)

# MUST add pickle5 to the import path because it will be imported by some
# raylet modules.

if "pickle5" in sys.modules:
    import pkg_resources
    try:
        version_info = pkg_resources.require("pickle5")
        version = tuple(int(n) for n in version_info[0].version.split("."))
        if version < (0, 0, 10):
            raise ImportError("You are using an old version of pickle5 "
                              "that leaks memory, please run "
                              "'pip install pickle5 -U' to upgrade")
    except pkg_resources.DistributionNotFound:
        logger.warning("You are using the 'pickle5' module, but "
                       "the exact version is unknown (possibly carried as "
                       "an internal component by another module). Please "
                       "make sure you are using pickle5 >= 0.0.10 because "
                       "previous versions may leak memory.")

if "OMP_NUM_THREADS" not in os.environ:
    logger.debug("[ray] Forcing OMP_NUM_THREADS=1 to avoid performance "
                 "degradation with many workers (issue #6998). You can "
                 "override this by explicitly setting OMP_NUM_THREADS.")
    os.environ["OMP_NUM_THREADS"] = "1"

# Add the directory containing pickle5 to the Python path so that we find the
# pickle5 version packaged with ray and not a pre-existing pickle5.
pickle5_path = os.path.join(
    os.path.abspath(os.path.dirname(__file__)), "pickle5_files")
sys.path.insert(0, pickle5_path)

# Importing psutil & setproctitle. Must be before ray._raylet is initialized.
thirdparty_files = os.path.join(
    os.path.abspath(os.path.dirname(__file__)), "thirdparty_files")
{'fraction_non_alphanumeric': 0.08799266727772685, 'fraction_numerical': 0.017720745493431105, 'mean_word_length': 2.930372148859544, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1167332', 'n_tokens_mistral': 988, 'n_tokens_neox': 957, 'n_words': 326}