<commit_before>
import os
import json
import subprocess
from subprocess import PIPE


def fetch_uris(sandbox_directory, uris):
    """
    Invoke the mesos-fetcher tool and download the given URIs.
    """

    # Build up the URIs
    fetcher_uris = []
    for uri in uris:
        uri_string = uri.value
        uri_string += "+"

        # Add the executable bit
        if uri.HasField("executable") and uri.executable:
            uri_string += "1"
        else:
            uri_string += "0"

        # Add the extraction bit
        if uri.extract:
            uri_string += "X"
        else:
            uri_string += "N"

        fetcher_uris.append(uri_string)

    fetcher_path = os.path.join(os.environ["MESOS_LIBEXEC_DIRECTORY"], "mesos-fetcher")
    proc = subprocess.Popen([fetcher_path], env={
        "MESOS_EXECUTOR_URIS": " ".join(fetcher_uris),
        "MESOS_WORK_DIRECTORY": sandbox_directory,
        "LD_LIBRARY_PATH": os.environ.get("LD_LIBRARY_PATH", "")
    })

    return proc.wait()<commit_msg>Log out the library path<commit_after>
import os
import json
import logging
import subprocess
from subprocess import PIPE

logger = logging.getLogger(__name__)


def fetch_uris(sandbox_directory, uris):
    """
    Invoke the mesos-fetcher tool and download the given URIs.
    """

    # Build up the URIs
    fetcher_uris = []
    for uri in uris:
        uri_string = uri.value
        uri_string += "+"

        # Add the executable bit
        if uri.HasField("executable") and uri.executable:
            uri_string += "1"
        else:
            uri_string += "0"

        # Add the extraction bit
        if uri.extract:
            uri_string += "X"
        else:
            uri_string += "N"

        fetcher_uris.append(uri_string)

    # Pass through the LD_LIBRARY_PATH
    library_path = os.environ.get("LD_LIBRARY_PATH", "")
    logger.info("LD_LIBRARY_PATH: %s", library_path)

    fetcher_path = os.path.join(os.environ["MESOS_LIBEXEC_DIRECTORY"], "mesos-fetcher")
    proc = subprocess.Popen([fetcher_path], env={
        "MESOS_EXECUTOR_URIS": " ".join(fetcher_uris),
        "MESOS_WORK_DIRECTORY": sandbox_directory,
        "LD_LIBRARY_PATH": library_path
    })

    return proc.wait()
{'fraction_non_alphanumeric': 0.10568741603224362, 'fraction_numerical': 0.0017913121361397223, 'mean_word_length': 2.5070643642072215, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6166556', 'n_tokens_mistral': 750, 'n_tokens_neox': 719, 'n_words': 164}