<commit_before>#include "primops.hh"
#include "eval-inline.hh"
#include "fetchGit.hh"
#include "download.hh"

#include <queue>
#include <nlohmann/json.hpp>

namespace nix {

const EvalState::FlakeRegistry & EvalState::getFlakeRegistry()
{
    std::call_once(_flakeRegistryInit, [&]()
    {
        _flakeRegistry = std::make_unique<FlakeRegistry>();

        if (!evalSettings.pureEval) {

            auto registryUri = "file:///home/eelco/Dev/gists/nix-flakes/registry.json";

            auto registryFile = getDownloader()->download(DownloadRequest(registryUri));

            auto json = nlohmann::json::parse(*registryFile.data);

            auto version = json.value("version", 0);
            if (version != 1)
                throw Error("flake registry '%s' has unsupported version %d", registryUri, version);

            auto flakes = json["flakes"];
            for (auto i = flakes.begin(); i != flakes.end(); ++i) {
                FlakeRegistry::Entry entry;
                entry.uri = i->value("uri", "");
                if (entry.uri.empty())
                    throw Error("invalid flake registry entry");
                _flakeRegistry->entries.emplace(i.key(), entry);
            }
        }
    });

    return *_flakeRegistry;
}

static void prim_flakeRegistry(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
    auto registry = state.getFlakeRegistry();

    state.mkAttrs(v, registry.entries.size());

    for (auto & entry : registry.entries) {
        auto vEntry = state.allocAttr(v, entry.first);
        state.mkAttrs(*vEntry, 2);
        mkString(*state.allocAttr(*vEntry, state.symbols.create("uri")), entry.second.uri);
        vEntry->attrs->sort();
    }

    v.attrs->sort();
}

static RegisterPrimOp r1("__flakeRegistry", 0, prim_flakeRegistry);

struct Flake
{
    std::string name;
    std::string description;
    Path path;
    std::set<std::string> requires;
    Value * vProvides; // FIXME: gc
};

static Flake fetchFlake(EvalState & state, const std::string & flakeUri)
{
    Flake flake;

    auto gitInfo = exportGit(state.store, flakeUri, {}, "", "source");

    state.store->assertStorePath(gitInfo.storePath);

    Value vInfo;
    state.evalFile(gitInfo.storePath + "/flake.nix", vInfo);

    state.forceAttrs(vInfo);

    if (auto name = vInfo.attrs->get(state.sName))
        flake.name = state.forceStringNoCtx(*(**name).value, *(**name).pos);
    else
        throw Error("flake lacks attribute 'name'");

    if (auto description = vInfo.attrs->get(state.sDescription))
        flake.description = state.forceStringNoCtx(*(**description).value, *(**description).pos);

    if (auto requires = vInfo.attrs->get(state.symbols.create("requires"))) {
        state.forceList(*(**requires).value, *(**requires).pos);
        for (unsigned int n = 0; n < (**requires).value->listSize(); ++n)
            flake.requires.insert(state.forceStringNoCtx(
                    *(**requires).value->listElems()[n], *(**requires).pos));
    }

    if (auto provides = vInfo.attrs->get(state.symbols.create("provides"))) {
        state.forceFunction(*(**provides).value, *(**provides).pos);
        flake.vProvides = (**provides).value;
    } else
        throw Error("flake lacks attribute 'provides'");

    return flake;
}

static std::map<std::string, Flake> resolveFlakes(EvalState & state, const StringSet & flakeUris)
{
    auto registry = state.getFlakeRegistry();

    std::map<std::string, Flake> done;
    std::queue<std::string> todo;
    for (auto & i : flakeUris) todo.push(i);

    while (!todo.empty()) {
        auto flakeUri = todo.front();
        todo.pop();
        if (done.count(flakeUri)) continue;

        auto flake = fetchFlake(state, flakeUri);

        for (auto & require : flake.requires) {
            auto i = registry.entries.find(require);
            if (i == registry.entries.end())
                throw Error("unknown flake '%s'", require);
            todo.push(i->second.uri);
        }<commit_msg>Introduce flake URIs<commit_after>#include "primops.hh"
#include "eval-inline.hh"
#include "fetchGit.hh"
#include "download.hh"

#include <queue>
#include <regex>
#include <nlohmann/json.hpp>

namespace nix {

const EvalState::FlakeRegistry & EvalState::getFlakeRegistry()
{
    std::call_once(_flakeRegistryInit, [&]()
    {
        _flakeRegistry = std::make_unique<FlakeRegistry>();

        if (!evalSettings.pureEval) {

            auto registryUri = "file:///home/eelco/Dev/gists/nix-flakes/registry.json";

            auto registryFile = getDownloader()->download(DownloadRequest(registryUri));

            auto json = nlohmann::json::parse(*registryFile.data);

            auto version = json.value("version", 0);
            if (version != 1)
                throw Error("flake registry '%s' has unsupported version %d", registryUri, version);

            auto flakes = json["flakes"];
            for (auto i = flakes.begin(); i != flakes.end(); ++i) {
                FlakeRegistry::Entry entry;
                entry.uri = i->value("uri", "");
                if (entry.uri.empty())
                    throw Error("invalid flake registry entry");
                _flakeRegistry->entries.emplace(i.key(), entry);
            }
        }
    });

    return *_flakeRegistry;
}

static void prim_flakeRegistry(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
    auto registry = state.getFlakeRegistry();

    state.mkAttrs(v, registry.entries.size());

    for (auto & entry : registry.entries) {
        auto vEntry = state.allocAttr(v, entry.first);
        state.mkAttrs(*vEntry, 2);
        mkString(*state.allocAttr(*vEntry, state.symbols.create("uri")), entry.second.uri);
        vEntry->attrs->sort();
    }

    v.attrs->sort();
}

static RegisterPrimOp r1("__flakeRegistry", 0, prim_flakeRegistry);

struct Flake
{
    std::string name;
    std::string description;
    Path path;
    std::set<std::string> requires;
    Value * vProvides; // FIXME: gc
};

std::regex flakeRegex("^flake:([a-zA-Z][a-zA-Z0-9_-]+)$");

static Path fetchFlake(EvalState & state, const std::string & flakeUri)
{
    std::smatch match;

    if (std::regex_match(flakeUri, match, flakeRegex)) {
        auto flakeName = match[1];
        auto registry = state.getFlakeRegistry();
        auto i = registry.entries.find(flakeName);
        if (i == registry.entries.end())
            throw Error("unknown flake '%s'", flakeName);
        return fetchFlake(state, i->second.uri);
    }

    else if (hasPrefix(flakeUri, "/") || hasPrefix(flakeUri, "git://")) {
        auto gitInfo = exportGit(state.store, flakeUri, {}, "", "source");
        return gitInfo.storePath;
    }

    else
        throw Error("unsupported flake URI '%s'", flakeUri);
}

static Flake getFlake(EvalState & state, const std::string & flakeUri)
{
    auto flakePath = fetchFlake(state, flakeUri);
    state.store->assertStorePath(flakePath);

    Flake flake;

    Value vInfo;
    state.evalFile(flakePath + "/flake.nix", vInfo);

    state.forceAttrs(vInfo);

    if (auto name = vInfo.attrs->get(state.sName))
        flake.name = state.forceStringNoCtx(*(**name).value, *(**name).pos);
    else
        throw Error("flake lacks attribute 'name'");

    if (auto description = vInfo.attrs->get(state.sDescription))
        flake.description = state.forceStringNoCtx(*(**description).value, *(**description).pos);

    if (auto requires = vInfo.attrs->get(state.symbols.create("requires"))) {
        state.forceList(*(**requires).value, *(**requires).pos);
        for (unsigned int n = 0; n < (**requires).value->listSize(); ++n)
            flake.requires.insert(state.forceStringNoCtx(
                    *(**requires).value->listElems()[n], *(**requires).pos));
    }

    if (auto provides = vInfo.attrs->get(state.symbols.create("provides"))) {
        state.forceFunction(*(**provides).value, *(**provides).pos);
        flake.vProvides = (**provides).value;
    } else
        throw Error("flake lacks attribute 'provides'");

    return flake;
}

static std::map<std::string, Flake> resolveFlakes(EvalState & state, const StringSet & flakeUris)
{
    std::map<std::string, Flake> done;
    std::queue<std::string> todo;
    for (auto & i : flakeUris) todo.push(i);

    while (!todo.empty()) {
        auto flakeUri = todo.front();
        todo.pop();
        if (done.count(flakeUri)) continue;

        auto flake = getFlake(state, flakeUri);

        for (auto & require : flake.requires)
            todo.push(require);
{'fraction_non_alphanumeric': 0.1584622597280825, 'fraction_numerical': 0.0017580872011251757, 'mean_word_length': 2.8839326354119255, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 46, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2618479', 'n_tokens_mistral': 2736, 'n_tokens_neox': 2632, 'n_words': 537}