<commit_before>#include "flakeref.hh"

#include <regex>

namespace nix {

// A Git ref (i.e. branch or tag name).
const static std::string refRegex = "[a-zA-Z][a-zA-Z0-9_.-]*"; // FIXME: check

// A Git revision (a SHA-1 commit hash).
const static std::string revRegexS = "[0-9a-fA-F]{40}";
std::regex revRegex(revRegexS, std::regex::ECMAScript);

// A Git ref or revision.
const static std::string revOrRefRegex = "(?:(" + revRegexS + ")|(" + refRegex + "))";

// A rev ("e72daba8250068216d79d2aeef40d4d95aff6666"), or a ref
// optionally followed by a rev (e.g. "master" or
// "master/e72daba8250068216d79d2aeef40d4d95aff6666").
const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegex + ")(?:/(" + revRegexS + "))?))";

const static std::string flakeId = "[a-zA-Z][a-zA-Z0-9_-]*";

// GitHub references.
const static std::string ownerRegex = "[a-zA-Z][a-zA-Z0-9_-]*";
const static std::string repoRegex = "[a-zA-Z][a-zA-Z0-9_-]*";

// URI stuff.
const static std::string schemeRegex = "(?:http|https|ssh|git|file)";
const static std::string authorityRegex = "[a-zA-Z0-9._~-]*";
const static std::string segmentRegex = "[a-zA-Z0-9._~-]+";
const static std::string pathRegex = "/?" + segmentRegex + "(?:/" + segmentRegex + ")*";
const static std::string paramRegex = "[a-z]+=[a-zA-Z0-9._-]*";

FlakeRef::FlakeRef(const std::string & uri)
{
    // FIXME: could combine this into one regex.

    static std::regex flakeRegex(
        "(?:flake:)?(" + flakeId + ")(?:/(?:" + refAndOrRevRegex + "))?",
        std::regex::ECMAScript);

    static std::regex githubRegex(
        "github:(" + ownerRegex + ")/(" + repoRegex + ")(?:/" + revOrRefRegex + ")?",
        std::regex::ECMAScript);

    static std::regex uriRegex(
        "((" + schemeRegex + "):" +
        "(?://(" + authorityRegex + "))?" +
        "(" + pathRegex + "))" +
        "(?:[?](" + paramRegex + "(?:&" + paramRegex + ")*))?",
        std::regex::ECMAScript);

    static std::regex refRegex2(refRegex, std::regex::ECMAScript);

    std::cmatch match;
    if (std::regex_match(uri.c_str(), match, flakeRegex)) {
        IsFlakeId d;
        d.id = match[1];
        if (match[2].matched)
            d.rev = Hash(match[2], htSHA1);
        else if (match[3].matched) {
            d.ref = match[3];
            if (match[4].matched)
                d.rev = Hash(match[4], htSHA1);
        }
        data = d;
    }

    else if (std::regex_match(uri.c_str(), match, githubRegex)) {
        IsGitHub d;
        d.owner = match[1];
        d.repo = match[2];
        if (match[3].matched)
            d.rev = Hash(match[3], htSHA1);
        else if (match[4].matched) {
            d.ref = match[4];
        }
        data = d;
    }

    else if (std::regex_match(uri.c_str(), match, uriRegex) && hasSuffix(match[4], ".git")) {
        IsGit d;
        d.uri = match[1];
        for (auto & param : tokenizeString<Strings>(match[5], "&")) {
            auto n = param.find('=');<commit_msg>Interpret all file:// URIs as Git repositories<commit_after>#include "flakeref.hh"

#include <regex>

namespace nix {

// A Git ref (i.e. branch or tag name).
const static std::string refRegex = "[a-zA-Z][a-zA-Z0-9_.-]*"; // FIXME: check

// A Git revision (a SHA-1 commit hash).
const static std::string revRegexS = "[0-9a-fA-F]{40}";
std::regex revRegex(revRegexS, std::regex::ECMAScript);

// A Git ref or revision.
const static std::string revOrRefRegex = "(?:(" + revRegexS + ")|(" + refRegex + "))";

// A rev ("e72daba8250068216d79d2aeef40d4d95aff6666"), or a ref
// optionally followed by a rev (e.g. "master" or
// "master/e72daba8250068216d79d2aeef40d4d95aff6666").
const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegex + ")(?:/(" + revRegexS + "))?))";

const static std::string flakeId = "[a-zA-Z][a-zA-Z0-9_-]*";

// GitHub references.
const static std::string ownerRegex = "[a-zA-Z][a-zA-Z0-9_-]*";
const static std::string repoRegex = "[a-zA-Z][a-zA-Z0-9_-]*";

// URI stuff.
const static std::string schemeRegex = "(?:http|https|ssh|git|file)";
const static std::string authorityRegex = "[a-zA-Z0-9._~-]*";
const static std::string segmentRegex = "[a-zA-Z0-9._~-]+";
const static std::string pathRegex = "/?" + segmentRegex + "(?:/" + segmentRegex + ")*";
const static std::string paramRegex = "[a-z]+=[a-zA-Z0-9._-]*";

FlakeRef::FlakeRef(const std::string & uri)
{
    // FIXME: could combine this into one regex.

    static std::regex flakeRegex(
        "(?:flake:)?(" + flakeId + ")(?:/(?:" + refAndOrRevRegex + "))?",
        std::regex::ECMAScript);

    static std::regex githubRegex(
        "github:(" + ownerRegex + ")/(" + repoRegex + ")(?:/" + revOrRefRegex + ")?",
        std::regex::ECMAScript);

    static std::regex uriRegex(
        "((" + schemeRegex + "):" +
        "(?://(" + authorityRegex + "))?" +
        "(" + pathRegex + "))" +
        "(?:[?](" + paramRegex + "(?:&" + paramRegex + ")*))?",
        std::regex::ECMAScript);

    static std::regex refRegex2(refRegex, std::regex::ECMAScript);

    std::cmatch match;
    if (std::regex_match(uri.c_str(), match, flakeRegex)) {
        IsFlakeId d;
        d.id = match[1];
        if (match[2].matched)
            d.rev = Hash(match[2], htSHA1);
        else if (match[3].matched) {
            d.ref = match[3];
            if (match[4].matched)
                d.rev = Hash(match[4], htSHA1);
        }
        data = d;
    }

    else if (std::regex_match(uri.c_str(), match, githubRegex)) {
        IsGitHub d;
        d.owner = match[1];
        d.repo = match[2];
        if (match[3].matched)
            d.rev = Hash(match[3], htSHA1);
        else if (match[4].matched) {
            d.ref = match[4];
        }
        data = d;
    }

    else if (std::regex_match(uri.c_str(), match, uriRegex)
        && (match[2] == "file" || hasSuffix(match[4], ".git")))
    {
        IsGit d;
        d.uri = match[1];
        for (auto & param : tokenizeString<Strings>(match[5], "&")) {
            auto n = param.find('=');
{'fraction_non_alphanumeric': 0.21032599702134702, 'fraction_numerical': 0.0289591262617905, 'mean_word_length': 2.789341692789969, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2467558', 'n_tokens_mistral': 2278, 'n_tokens_neox': 2168, 'n_words': 447}