<commit_before>import logging
import os
import shutil
import subprocess
import time

log = logging.getLogger(__name__)


def checkout_repo(repo_url, dest_path, branch):
    # if os.path.isdir(path):
    #     p = subprocess.Popen('git status', shell=True, cwd=path)
    #     if p.wait() == 128:
    #         log.info("Repo at %s appears corrupt; removing",
    #                  branch)
    #         shutil.rmtree(path)

    if not os.path.isdir(dest_path):
        log.info("Cloning %s %s from upstream", repo_url, branch)
        log.info(
            subprocess.check_output(('git', 'clone', '--branch', branch,
                                     repo_url, dest_path),
                                    cwd=os.path.dirname(dest_path))
        )
    elif time.time() - os.stat('/etc/passwd').st_mtime > 60:
        # only do this at most once per minute
        log.info("Fetching %s from upstream", branch)
        log.info(
            subprocess.check_output(('git', 'fetch', '-p', 'origin'),
                                    cwd=dest_path)
        )
        log.info(
            subprocess.check_output(('touch', dest_path))
        )
    else:
        log.info("%s was just updated; assuming it is current", branch)

    # This try/except block will notice if the requested branch doesn't<commit_msg>Improve missing branch detection and logging<commit_after>import logging
import os
import shutil
import subprocess
import time

log = logging.getLogger(__name__)


def checkout_repo(repo_url, dest_path, branch):
    # if os.path.isdir(path):
    #     p = subprocess.Popen('git status', shell=True, cwd=path)
    #     if p.wait() == 128:
    #         log.info("Repo at %s appears corrupt; removing",
    #                  branch)
    #         shutil.rmtree(path)

    if not os.path.isdir(dest_path):
        log.info("Cloning %s %s from upstream", repo_url, branch)
        proc = subprocess.Popen(
            ('git', 'clone', '--branch', branch, repo_url, dest_path),
            cwd=os.path.dirname(dest_path),
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT)
        not_found_str = "Remote branch %s not found" % branch
        out = proc.stdout.read()
        if proc.wait() != 0:
            log.error(out)
            if not_found_str in out:
                raise BranchNotFoundError(branch, repo_url)
            else:
                raise RuntimeError("git clone failed!")
    elif time.time() - os.stat('/etc/passwd').st_mtime > 60:
        # only do this at most once per minute
        log.info("Fetching %s from upstream", branch)
        out = subprocess.check_output(('git', 'fetch', '-p', 'origin'),
                                      cwd=dest_path)
        if out:
            log.info(out)
        out = subprocess.check_output(('touch', dest_path))
        if out:
            log.info(out)
    else:
        log.info("%s was just updated; assuming it is current", branch)

    # This try/except block will notice if the requested branch doesn't
{'fraction_non_alphanumeric': 0.11692613448161643, 'fraction_numerical': 0.00364359059291156, 'mean_word_length': 2.0019880715705765, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5076636', 'n_tokens_mistral': 915, 'n_tokens_neox': 881, 'n_words': 234}