<commit_before>#include "fetchGit.hh"
#include "primops.hh"
#include "eval-inline.hh"
#include "download.hh"
#include "store-api.hh"
#include "pathlocks.hh"
#include "hash.hh"

#include <sys/time.h>

#include <regex>

#include <nlohmann/json.hpp>

using namespace std::string_literals;

namespace nix {

extern std::regex revRegex;

GitInfo exportGit(ref<Store> store, std::string uri,
    std::optional<std::string> ref,
    std::optional<Hash> rev,
    const std::string & name)
{
    assert(!rev || rev->type == htSHA1);

    bool isLocal = hasPrefix(uri, "/") && pathExists(uri + "/.git");

    // If this is a local directory (but not a file:// URI) and no ref
    // or revision is given, then allow the use of an unclean working
    // tree.
    if (!ref && !rev && isLocal) {

        bool clean = true;

        try {
            runProgram("git", true, { "-C", uri, "diff-index", "--quiet", "HEAD", "--" });
        } catch (ExecError e) {
            if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
            clean = false;
        }

        if (!clean) {

            /* This is an unclean working tree. So copy all tracked
               files. */

            GitInfo gitInfo;
            gitInfo.ref = "HEAD";

            auto files = tokenizeString<std::set<std::string>>(
                runProgram("git", true, { "-C", uri, "ls-files", "-z" }), "\0"s);

            PathFilter filter = [&](const Path & p) -> bool {
                assert(hasPrefix(p, uri));
                std::string file(p, uri.size() + 1);

                auto st = lstat(p);

                if (S_ISDIR(st.st_mode)) {
                    auto prefix = file + "/";
                    auto i = files.lower_bound(prefix);
                    return i != files.end() && hasPrefix(*i, prefix);
                }

                return files.count(file);
            };

            gitInfo.storePath = store->addToStore("source", uri, true, htSHA256, filter);
            gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", uri, "rev-list", "--count", "HEAD" }));

            return gitInfo;
        }
    }

    if (!ref) ref = isLocal ? "HEAD" : "master";

    // Don't clone file:// URIs (but otherwise treat them the same as
    // remote URIs, i.e. don't use the working tree or HEAD).
    static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing
    if (!forceHttp && hasPrefix(uri, "file://")) {
        uri = std::string(uri, 7);
        isLocal = true;
    }

    deletePath(getCacheDir() + "/nix/git");

    Path cacheDir = getCacheDir() + "/nix/gitv2/" + hashString(htSHA256, uri).to_string(Base32, false);
    Path repoDir;

    if (isLocal) {

        if (!rev)
            rev = Hash(chomp(runProgram("git", true, { "-C", uri, "rev-parse", *ref })), htSHA1);

        if (!pathExists(cacheDir))
            createDirs(cacheDir);

        repoDir = uri;

    } else {

        repoDir = cacheDir;

        if (!pathExists(cacheDir)) {
            createDirs(dirOf(cacheDir));
            runProgram("git", true, { "init", "--bare", repoDir });
        }

        Path localRefFile = repoDir + "/refs/heads/" + *ref;

        bool doFetch;
        time_t now = time(0);

        /* If a rev was specified, we need to fetch if it's not in the
           repo. */
        if (rev) {
            try {
                runProgram("git", true, { "-C", repoDir, "cat-file", "-e", rev->gitRev() });
                doFetch = false;
            } catch (ExecError & e) {
                if (WIFEXITED(e.status)) {
                    doFetch = true;
                } else {
                    throw;
                }
            }
        } else {
            /* If the local ref is older than ‘tarball-ttl’ seconds, do a
               git fetch to update the local ref to the remote ref. */
            struct stat st;
            doFetch = stat(localRefFile.c_str(), &st) != 0 ||
                st.st_mtime + settings.tarballTtl <= now;
        }

        if (doFetch) {
            Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", uri));

            // FIXME: git stderr messes up our progress indicator, so
            // we're using --quiet for now. Should process its stderr.
            runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) });

            struct timeval times[2];
            times[0].tv_sec = now;
            times[0].tv_usec = 0;
            times[1].tv_sec = now;
            times[1].tv_usec = 0;

            utimes(localRefFile.c_str(), times);
        }

        if (!rev)
            rev = Hash(chomp(readFile(localRefFile)), htSHA1);
    }

    // FIXME: check whether rev is an ancestor of ref.<commit_msg>fetchGit: Don't barf if we can't update our Git clone<commit_after>#include "fetchGit.hh"
#include "primops.hh"
#include "eval-inline.hh"
#include "download.hh"
#include "store-api.hh"
#include "pathlocks.hh"
#include "hash.hh"

#include <sys/time.h>

#include <regex>

#include <nlohmann/json.hpp>

using namespace std::string_literals;

namespace nix {

extern std::regex revRegex;

GitInfo exportGit(ref<Store> store, std::string uri,
    std::optional<std::string> ref,
    std::optional<Hash> rev,
    const std::string & name)
{
    assert(!rev || rev->type == htSHA1);

    bool isLocal = hasPrefix(uri, "/") && pathExists(uri + "/.git");

    // If this is a local directory (but not a file:// URI) and no ref
    // or revision is given, then allow the use of an unclean working
    // tree.
    if (!ref && !rev && isLocal) {

        bool clean = true;

        try {
            runProgram("git", true, { "-C", uri, "diff-index", "--quiet", "HEAD", "--" });
        } catch (ExecError e) {
            if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
            clean = false;
        }

        if (!clean) {

            /* This is an unclean working tree. So copy all tracked
               files. */

            GitInfo gitInfo;
            gitInfo.ref = "HEAD";

            auto files = tokenizeString<std::set<std::string>>(
                runProgram("git", true, { "-C", uri, "ls-files", "-z" }), "\0"s);

            PathFilter filter = [&](const Path & p) -> bool {
                assert(hasPrefix(p, uri));
                std::string file(p, uri.size() + 1);

                auto st = lstat(p);

                if (S_ISDIR(st.st_mode)) {
                    auto prefix = file + "/";
                    auto i = files.lower_bound(prefix);
                    return i != files.end() && hasPrefix(*i, prefix);
                }

                return files.count(file);
            };

            gitInfo.storePath = store->addToStore("source", uri, true, htSHA256, filter);
            gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", uri, "rev-list", "--count", "HEAD" }));

            return gitInfo;
        }
    }

    if (!ref) ref = isLocal ? "HEAD" : "master";

    // Don't clone file:// URIs (but otherwise treat them the same as
    // remote URIs, i.e. don't use the working tree or HEAD).
    static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing
    if (!forceHttp && hasPrefix(uri, "file://")) {
        uri = std::string(uri, 7);
        isLocal = true;
    }

    deletePath(getCacheDir() + "/nix/git");

    Path cacheDir = getCacheDir() + "/nix/gitv2/" + hashString(htSHA256, uri).to_string(Base32, false);
    Path repoDir;

    if (isLocal) {

        if (!rev)
            rev = Hash(chomp(runProgram("git", true, { "-C", uri, "rev-parse", *ref })), htSHA1);

        if (!pathExists(cacheDir))
            createDirs(cacheDir);

        repoDir = uri;

    } else {

        repoDir = cacheDir;

        if (!pathExists(cacheDir)) {
            createDirs(dirOf(cacheDir));
            runProgram("git", true, { "init", "--bare", repoDir });
        }

        Path localRefFile = repoDir + "/refs/heads/" + *ref;

        bool doFetch;
        time_t now = time(0);

        /* If a rev was specified, we need to fetch if it's not in the
           repo. */
        if (rev) {
            try {
                runProgram("git", true, { "-C", repoDir, "cat-file", "-e", rev->gitRev() });
                doFetch = false;
            } catch (ExecError & e) {
                if (WIFEXITED(e.status)) {
                    doFetch = true;
                } else {
                    throw;
                }
            }
        } else {
            /* If the local ref is older than ‘tarball-ttl’ seconds, do a
               git fetch to update the local ref to the remote ref. */
            struct stat st;
            doFetch = stat(localRefFile.c_str(), &st) != 0 ||
                st.st_mtime + settings.tarballTtl <= now;
        }

        if (doFetch) {
            Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", uri));

            // FIXME: git stderr messes up our progress indicator, so
            // we're using --quiet for now. Should process its stderr.
            try {
                runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) });
            } catch (Error & e) {
                if (!pathExists(localRefFile)) throw;
                warn("could not update local clone of Git repository '%s'; continuing with the most recent version", uri);
            }

            struct timeval times[2];
            times[0].tv_sec = now;
            times[0].tv_usec = 0;
            times[1].tv_sec = now;
            times[1].tv_usec = 0;

            utimes(localRefFile.c_str(), times);
        }

        if (!rev)
            rev = Hash(chomp(readFile(localRefFile)), htSHA1);
    }

    // FIXME: check whether rev is an ancestor of ref.
{'fraction_non_alphanumeric': 0.15175215845606907, 'fraction_numerical': 0.005281868969019807, 'mean_word_length': 1.7766497461928934, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2246452', 'n_tokens_mistral': 3120, 'n_tokens_neox': 2953, 'n_words': 833}