<commit_before>#include "primops.hh"
#include "eval-inline.hh"
#include "store-api.hh"
#include "fetchers.hh"
#include "filetransfer.hh"
#include "registry.hh"

#include <ctime>
#include <iomanip>

namespace nix {

void emitTreeAttrs(
    EvalState & state,
    const fetchers::Tree & tree,
    const fetchers::Input & input,
    Value & v,
    bool emptyRevFallback)
{
    assert(input.isImmutable());

    state.mkAttrs(v, 8);

    auto storePath = state.store->printStorePath(tree.storePath);

    mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));

    // FIXME: support arbitrary input attributes.

    auto narHash = input.getNarHash();
    assert(narHash);
    mkString(*state.allocAttr(v, state.symbols.create("narHash")),
        narHash->to_string(SRI, true));

    if (auto rev = input.getRev()) {
        mkString(*state.allocAttr(v, state.symbols.create("rev")), rev->gitRev());
        mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev->gitShortRev());
    } else if (emptyRevFallback) {
        // Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev
        auto emptyHash = Hash(htSHA1);
        mkString(*state.allocAttr(v, state.symbols.create("rev")), emptyHash.gitRev());
        mkString(*state.allocAttr(v, state.symbols.create("shortRev")), emptyHash.gitShortRev());
    }

    if (input.getType() == "git")
        mkBool(*state.allocAttr(v, state.symbols.create("submodules")),
            fetchers::maybeGetBoolAttr(input.attrs, "submodules").value_or(false));

    if (auto revCount = input.getRevCount())
        mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount);
    else if (emptyRevFallback)
        mkInt(*state.allocAttr(v, state.symbols.create("revCount")), 0);

    if (auto lastModified = input.getLastModified()) {
        mkInt(*state.allocAttr(v, state.symbols.create("lastModified")), *lastModified);
        mkString(*state.allocAttr(v, state.symbols.create("lastModifiedDate")),
            fmt("%s", std::put_time(std::gmtime(&*lastModified), "%Y%m%d%H%M%S")));
    }

    v.attrs->sort();
}

std::string fixURI(std::string uri, EvalState &state)
{
    state.checkURI(uri);
    return uri.find("://") != std::string::npos ? uri : "file://" + uri;
}

void addURI(EvalState &state, fetchers::Attrs &attrs, Symbol name, std::string v)
{
    string n(name);
    attrs.emplace(name, n == "url" ? fixURI(v, state) : v);
}

static void fetchTree(
    EvalState &state,
    const Pos &pos,
    Value **args,
    Value &v,
    const std::optional<std::string> type,
    bool emptyRevFallback = false
) {
    fetchers::Input input;
    PathSet context;

    state.forceValue(*args[0]);

    if (args[0]->type() == nAttrs) {
        state.forceAttrs(*args[0], pos);

        fetchers::Attrs attrs;

        for (auto & attr : *args[0]->attrs) {
            state.forceValue(*attr.value);
            if (attr.value->type() == nPath || attr.value->type() == nString)
                addURI(
                    state,
                    attrs,
                    attr.name,
                    state.coerceToString(*attr.pos, *attr.value, context, false, false)
                );
            else if (attr.value->type() == nString)
                addURI(state, attrs, attr.name, attr.value->string.s);
            else if (attr.value->type() == nBool)
                attrs.emplace(attr.name, Explicit<bool>{attr.value->boolean});
            else if (attr.value->type() == nInt)
                attrs.emplace(attr.name, uint64_t(attr.value->integer));
            else
                throw TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected",
                    attr.name, showType(*attr.value));
        }

        if (type)
            attrs.emplace("type", type.value());

        if (!attrs.count("type"))
            throw Error({
                .msg = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
                .errPos = pos
            });

        input = fetchers::Input::fromAttrs(std::move(attrs));
    } else {
        auto url = fixURI(state.coerceToString(pos, *args[0], context, false, false), state);

        if (type == "git") {
            fetchers::Attrs attrs;
            attrs.emplace("type", "git");
            attrs.emplace("url", url);
            input = fetchers::Input::fromAttrs(std::move(attrs));
        } else {
            input = fetchers::Input::fromURL(url);
        }
    }

    if (!evalSettings.pureEval && !input.isDirect())
        input = lookupInRegistries(state.store, input).first;

    if (evalSettings.pureEval && !input.isImmutable())
        throw Error("in pure evaluation mode, 'fetchTree' requires an immutable input, at %s", pos);

    auto [tree, input2] = input.fetch(state.store);<commit_msg>Allow scp-style uris in `fetchgit`<commit_after>#include "primops.hh"
#include "eval-inline.hh"
#include "store-api.hh"
#include "fetchers.hh"
#include "filetransfer.hh"
#include "registry.hh"

#include <ctime>
#include <iomanip>
#include <regex>

namespace nix {

void emitTreeAttrs(
    EvalState & state,
    const fetchers::Tree & tree,
    const fetchers::Input & input,
    Value & v,
    bool emptyRevFallback)
{
    assert(input.isImmutable());

    state.mkAttrs(v, 8);

    auto storePath = state.store->printStorePath(tree.storePath);

    mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));

    // FIXME: support arbitrary input attributes.

    auto narHash = input.getNarHash();
    assert(narHash);
    mkString(*state.allocAttr(v, state.symbols.create("narHash")),
        narHash->to_string(SRI, true));

    if (auto rev = input.getRev()) {
        mkString(*state.allocAttr(v, state.symbols.create("rev")), rev->gitRev());
        mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev->gitShortRev());
    } else if (emptyRevFallback) {
        // Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev
        auto emptyHash = Hash(htSHA1);
        mkString(*state.allocAttr(v, state.symbols.create("rev")), emptyHash.gitRev());
        mkString(*state.allocAttr(v, state.symbols.create("shortRev")), emptyHash.gitShortRev());
    }

    if (input.getType() == "git")
        mkBool(*state.allocAttr(v, state.symbols.create("submodules")),
            fetchers::maybeGetBoolAttr(input.attrs, "submodules").value_or(false));

    if (auto revCount = input.getRevCount())
        mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount);
    else if (emptyRevFallback)
        mkInt(*state.allocAttr(v, state.symbols.create("revCount")), 0);

    if (auto lastModified = input.getLastModified()) {
        mkInt(*state.allocAttr(v, state.symbols.create("lastModified")), *lastModified);
        mkString(*state.allocAttr(v, state.symbols.create("lastModifiedDate")),
            fmt("%s", std::put_time(std::gmtime(&*lastModified), "%Y%m%d%H%M%S")));
    }

    v.attrs->sort();
}

std::string fixURI(std::string uri, EvalState &state, const std::string & defaultScheme = "file")
{
    state.checkURI(uri);
    return uri.find("://") != std::string::npos ? uri : defaultScheme + "://" + uri;
}

std::string fixURIForGit(std::string uri, EvalState & state)
{
    static std::regex scp_uri("([^/].*)@(.*):(.*)");
    if (uri[0] != '/' && std::regex_match(uri, scp_uri))
        return fixURI(std::regex_replace(uri, scp_uri, "$1@$2/$3"), state, "ssh");
    else
        return fixURI(uri, state);
}

void addURI(EvalState &state, fetchers::Attrs &attrs, Symbol name, std::string v)
{
    string n(name);
    attrs.emplace(name, n == "url" ? fixURI(v, state) : v);
}

static void fetchTree(
    EvalState &state,
    const Pos &pos,
    Value **args,
    Value &v,
    const std::optional<std::string> type,
    bool emptyRevFallback = false
) {
    fetchers::Input input;
    PathSet context;

    state.forceValue(*args[0]);

    if (args[0]->type() == nAttrs) {
        state.forceAttrs(*args[0], pos);

        fetchers::Attrs attrs;

        for (auto & attr : *args[0]->attrs) {
            state.forceValue(*attr.value);
            if (attr.value->type() == nPath || attr.value->type() == nString)
                addURI(
                    state,
                    attrs,
                    attr.name,
                    state.coerceToString(*attr.pos, *attr.value, context, false, false)
                );
            else if (attr.value->type() == nString)
                addURI(state, attrs, attr.name, attr.value->string.s);
            else if (attr.value->type() == nBool)
                attrs.emplace(attr.name, Explicit<bool>{attr.value->boolean});
            else if (attr.value->type() == nInt)
                attrs.emplace(attr.name, uint64_t(attr.value->integer));
            else
                throw TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected",
                    attr.name, showType(*attr.value));
        }

        if (type)
            attrs.emplace("type", type.value());

        if (!attrs.count("type"))
            throw Error({
                .msg = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
                .errPos = pos
            });

        input = fetchers::Input::fromAttrs(std::move(attrs));
    } else {
        auto url = state.coerceToString(pos, *args[0], context, false, false);

        if (type == "git") {
            fetchers::Attrs attrs;
            attrs.emplace("type", "git");
            attrs.emplace("url", fixURIForGit(url, state));
            input = fetchers::Input::fromAttrs(std::move(attrs));
        } else {
            input = fetchers::Input::fromURL(fixURI(url, state));
        }
    }

    if (!evalSettings.pureEval && !input.isDirect())
        input = lookupInRegistries(state.store, input).first;

    if (evalSettings.pureEval && !input.isImmutable())
        throw Error("in pure evaluation mode, 'fetchTree' requires an immutable input, at %s", pos);

    auto [tree, input2] = input.fetch(state.store);
{'fraction_non_alphanumeric': 0.16204986149584488, 'fraction_numerical': 0.002770083102493075, 'mean_word_length': 2.7524127691165554, 'pattern_counts': {'":': 3, '<': 12, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4130300', 'n_tokens_mistral': 3225, 'n_tokens_neox': 3142, 'n_words': 646}