<commit_before>/*
 * Copyright (c) 2016, Two Sigma Open Source
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * * Redistributions of source code must retain the above copyright notice,
 *   this list of conditions and the following disclaimer.
 *
 * * Redistributions in binary form must reproduce the above copyright notice,
 *   this list of conditions and the following disclaimer in the documentation
 *   and/or other materials provided with the distribution.
 *
 * * Neither the name of git-meta nor the names of its
 *   contributors may be used to endorse or promote products derived from
 *   this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */
"use strict";

/**
 * This module contains the pre-recieve hook for a git-meta server.  Only
 the meta-repo should use this hook.
 */

const NodeGit = require("nodegit");
const GitUtil = require("./git_util");
const SubmoduleUtil = require("./submodule_util");
const split = require("split");
const co = require("co");
const path = require("path");
const assert = require("chai").assert;

const NOTES_REF = "refs/notes/git-meta/subrepo-check";
const SYNTHETIC_BRANCH_BASE = "refs/commits/";

/**
 * The identity function
 */
function identity(v) {
    return v;
}

    /**
     * (This has to be public so we can mock it for testing)
     * @param {NodeGit.Commit} commit
     */
exports.getSyntheticBranchForCommit = function(commit) {
    return SYNTHETIC_BRANCH_BASE + commit;
};

function *urlToLocalPath(repo, url) {
    assert.instanceOf(repo, NodeGit.Repository);
    assert.isString(url);

    const config = yield repo.config();
    let subrepoUrlBase = "";
    try {
        subrepoUrlBase =
              yield config.getStringBuf("gitmeta.subrepourlbase");
    } catch (e) {
        //It's OK for this to be missing, but nodegit lacks an
        //API that expresses this.

    }
    const subrepoRootPath =
        yield config.getStringBuf("gitmeta.subreporootpath");
    let subrepoSuffix = "";
    try {
        subrepoSuffix = yield config.getStringBuf("gitmeta.subreposuffix");
    } catch (e) {
        //It's OK for this to be missing, but nodegit lacks an
        //API that expresses this.
    }
    if (!url.startsWith(subrepoUrlBase)) {
        throw "Your git configuration gitmeta.subrepoUrlBase, '" +
            subrepoUrlBase + "', must be a prefix of all submodule " +
            "urls.  Submodule url '" + url + "' fails.";
    }
    const remotePath = url.slice(subrepoUrlBase.length);
    const localPath = path.join(subrepoRootPath, remotePath + subrepoSuffix);
    if (localPath[0] === "/") {
        return localPath;
    } else {
        return path.normalize(path.join(repo.path(), localPath));
    }
}

/**
 * Check that a synthetic branch exists for a given submodule
 * at a given commit.
 * @async
 * @param {NodeGit.Repostory} repo The meta repository
 * @param {NodeGit.TreeEntry} submoduleEntry the submodule's tree entry
 * in the meta tree.
 */
function* checkSubmodule(repo, metaCommit, submoduleEntry) {
    assert.instanceOf(repo, NodeGit.Repository);
    assert.instanceOf(submoduleEntry, NodeGit.TreeEntry);

    const submodulePath = submoduleEntry.path();

    const submodules = yield SubmoduleUtil.getSubmodulesForCommit(repo,
                                                                  metaCommit);
    const url = submodules[submodulePath].url;
    const localPath = yield *urlToLocalPath(repo, url);
    const submoduleRepo = yield NodeGit.Repository.open(localPath);
    const submoduleCommitId = submoduleEntry.id();
    const branch = exports.getSyntheticBranchForCommit(submoduleCommitId);
    try {
        const subrepoCommit =
            yield submoduleRepo.getReferenceCommit(branch);
        return subrepoCommit.id().equal(submoduleEntry.id());
    } catch (e) {
        console.error("Could not look up ", branch, " in ", localPath,
                      ": ", e);
        return false;
    }
}

function* checkSubmodules(repo, commit) {
    assert.instanceOf(repo, NodeGit.Repository);
    assert.instanceOf(commit, NodeGit.Commit);
    const getChanges = SubmoduleUtil.getSubmoduleChanges;
    const changes = yield getChanges(repo, commit);
    const allChanges = [changes.added, changes.changed];
    const result = allChanges.map(function *(changeSet) {
        const result = Array.from(changeSet).map(function *(path) {
            const entry = yield commit.getEntry(path);
            return yield *checkSubmodule(repo, commit, entry);
        });
        return (yield result).every(identity);
    });
    return (yield result).every(identity);
}

/**
 * Recurse over the parents of a commit, checking each for git-meta's
 * invariants. Returns when all (transitive) parents have already been
 * marked as checked, or a failure is found.
 *
 * Returns true if the check passes on all branches; false if any
 * fail.
 *
 * @async
 * @param {NodeGit.Repostory} repo The meta repository
 * @param {NodeGit.Commit} commit The meta branch's commit to check
 * @param {String} oldSha the previous (known-good) value of this ref
 * @param {Object} handled the commit ids that have been already
 * processed (and the result of processing them).
 */
function* parentLoop(repo, commit, oldSha, handled) {
    assert.instanceOf(repo, NodeGit.Repository);
    assert.instanceOf(commit, NodeGit.Commit);
    assert.isString(oldSha);
    assert.isObject(handled);

    if (commit.id() in handled)  {
        return handled[commit.id()];<commit_msg>Load submodule configurations only once-per-check<commit_after>/*
 * Copyright (c) 2016, Two Sigma Open Source
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * * Redistributions of source code must retain the above copyright notice,
 *   this list of conditions and the following disclaimer.
 *
 * * Redistributions in binary form must reproduce the above copyright notice,
 *   this list of conditions and the following disclaimer in the documentation
 *   and/or other materials provided with the distribution.
 *
 * * Neither the name of git-meta nor the names of its
 *   contributors may be used to endorse or promote products derived from
 *   this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */
"use strict";

/**
 * This module contains the pre-recieve hook for a git-meta server.  Only
 the meta-repo should use this hook.
 */

const NodeGit = require("nodegit");
const GitUtil = require("./git_util");
const SubmoduleUtil = require("./submodule_util");
const split = require("split");
const co = require("co");
const path = require("path");
const assert = require("chai").assert;

const NOTES_REF = "refs/notes/git-meta/subrepo-check";
const SYNTHETIC_BRANCH_BASE = "refs/commits/";

/**
 * The identity function
 */
function identity(v) {
    return v;
}

    /**
     * (This has to be public so we can mock it for testing)
     * @param {NodeGit.Commit} commit
     */
exports.getSyntheticBranchForCommit = function(commit) {
    return SYNTHETIC_BRANCH_BASE + commit;
};

function *urlToLocalPath(repo, url) {
    assert.instanceOf(repo, NodeGit.Repository);
    assert.isString(url);

    const config = yield repo.config();
    let subrepoUrlBase = "";
    try {
        subrepoUrlBase =
              yield config.getStringBuf("gitmeta.subrepourlbase");
    } catch (e) {
        //It's OK for this to be missing, but nodegit lacks an
        //API that expresses this.

    }
    const subrepoRootPath =
        yield config.getStringBuf("gitmeta.subreporootpath");
    let subrepoSuffix = "";
    try {
        subrepoSuffix = yield config.getStringBuf("gitmeta.subreposuffix");
    } catch (e) {
        //It's OK for this to be missing, but nodegit lacks an
        //API that expresses this.
    }
    if (!url.startsWith(subrepoUrlBase)) {
        throw "Your git configuration gitmeta.subrepoUrlBase, '" +
            subrepoUrlBase + "', must be a prefix of all submodule " +
            "urls.  Submodule url '" + url + "' fails.";
    }
    const remotePath = url.slice(subrepoUrlBase.length);
    const localPath = path.join(subrepoRootPath, remotePath + subrepoSuffix);
    if (localPath[0] === "/") {
        return localPath;
    } else {
        return path.normalize(path.join(repo.path(), localPath));
    }
}

/**
 * Check that a synthetic branch exists for a given submodule
 * at a given commit.
 * @async
 * @param {NodeGit.Repostory} repo The meta repository
 * @param {NodeGit.TreeEntry} submoduleEntry the submodule's tree entry
 * @param {String}            url the configured URL of the submodule
 * in the meta tree.
 */
function* checkSubmodule(repo, metaCommit, submoduleEntry, url) {
    assert.instanceOf(repo, NodeGit.Repository);
    assert.instanceOf(submoduleEntry, NodeGit.TreeEntry);

    const localPath = yield *urlToLocalPath(repo, url);
    const submoduleRepo = yield NodeGit.Repository.open(localPath);
    const submoduleCommitId = submoduleEntry.id();
    const branch = exports.getSyntheticBranchForCommit(submoduleCommitId);
    try {
        const subrepoCommit =
            yield submoduleRepo.getReferenceCommit(branch);
        return subrepoCommit.id().equal(submoduleEntry.id());
    } catch (e) {
        console.error("Could not look up ", branch, " in ", localPath,
                      ": ", e);
        return false;
    }
}

function* checkSubmodules(repo, commit) {
    assert.instanceOf(repo, NodeGit.Repository);
    assert.instanceOf(commit, NodeGit.Commit);

    const submodules = yield SubmoduleUtil.getSubmodulesForCommit(repo,
                                                                  commit);
    const getChanges = SubmoduleUtil.getSubmoduleChanges;
    const changes = yield getChanges(repo, commit);
    const allChanges = [changes.added, changes.changed];
    const result = allChanges.map(function *(changeSet) {
        const result = Array.from(changeSet).map(function *(path) {
            const entry = yield commit.getEntry(path);
            const submodulePath = entry.path();
            const url = submodules[submodulePath].url;
            return yield *checkSubmodule(repo, commit, entry, url);
        });
        return (yield result).every(identity);
    });
    return (yield result).every(identity);
}

/**
 * Recurse over the parents of a commit, checking each for git-meta's
 * invariants. Returns when all (transitive) parents have already been
 * marked as checked, or a failure is found.
 *
 * Returns true if the check passes on all branches; false if any
 * fail.
 *
 * @async
 * @param {NodeGit.Repostory} repo The meta repository
 * @param {NodeGit.Commit} commit The meta branch's commit to check
 * @param {String} oldSha the previous (known-good) value of this ref
 * @param {Object} handled the commit ids that have been already
 * processed (and the result of processing them).
 */
function* parentLoop(repo, commit, oldSha, handled) {
    assert.instanceOf(repo, NodeGit.Repository);
    assert.instanceOf(commit, NodeGit.Commit);
    assert.isString(oldSha);
    assert.isObject(handled);

    if (commit.id() in handled)  {
        return handled[commit.id()];
{'fraction_non_alphanumeric': 0.0967617281088183, 'fraction_numerical': 0.0007728572532653219, 'mean_word_length': 3.303292317924842, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1673880', 'n_tokens_mistral': 3651, 'n_tokens_neox': 3505, 'n_words': 1252}