<commit_before>var B = require("buster-core");
var when = require("when");
var fs = require("fs");
var glob = require("buster-glob").glob;
var Path = require("path");
var fileEtag = require("./file-etag");

function addUnique(arr1, arr2) {
    arr1 = arr1 || [];
    (arr2 || []).forEach(function (item) {
        if (arr1.indexOf(item) < 0) { arr1.push(item); }
    });
    return arr1;
}

function relativePath(rootPath, path) {
    return path.replace(rootPath.replace(/\/?$/, "/"), "");
}

function absolutePath(rootPath, path) {
    return Path.resolve(rootPath, path);
}

function outsideRoot(rootPath, path) {
    return path.indexOf(rootPath) < 0;
}

function outsideRootError(rootPath, paths) {
    var offendingPaths = paths.filter(B.partial(outsideRoot, rootPath));
    if (offendingPaths.length === 0) { return; }
    var plural = offendingPaths.length > 1 ? "Some paths are " : "A path is ";
    var offending = offendingPaths.map(B.partial(Path.relative, rootPath));

    return new Error(plural + "outside the project root. Set rootPath to " +
                     "the desired root to refer to paths outside of " +
                     this.rootPath + ".\n  " + offending.join("\n  "));
}

function resolvePaths(rs, paths, callback) {
    glob(paths.map(B.partial(absolutePath, rs.rootPath)), function (err, ms) {
        err = err || outsideRootError(rs.rootPath, ms);
        if (err) { return callback.call(rs, err); }
        ms = ms.map(B.partial(relativePath, rs.rootPath));
        ms = addUnique(ms, rs.matchPaths(paths));
        callback.call(rs, err, ms);
    });
}

function fileReader(fileName) {
    return function () {
        var d = when.defer();
        fs.readFile(fileName, this.encoding, function (err, data) {
            if (err) { return d.resolver.reject(err); }
            d.resolver.resolve(data);
        });
        return d.promise;
    };
}

function prepareResource(rootPath, path, resource) {
    var d = when.defer();
    var fileName = absolutePath(rootPath, path);
    resource.content = fileReader(fileName);
    fileEtag.add(fileName, resource, function (err) {<commit_msg>Minor fix to support glob 3.0<commit_after>var B = require("buster-core");
var when = require("when");
var fs = require("fs");
var glob = require("buster-glob").glob;
var Path = require("path");
var fileEtag = require("./file-etag");

function addUnique(arr1, arr2) {
    arr1 = arr1 || [];
    (arr2 || []).forEach(function (item) {
        if (arr1.indexOf(item) < 0) { arr1.push(item); }
    });
    return arr1;
}

function relativePath(rootPath, path) {
    return path.replace(rootPath.replace(/\/?$/, "/"), "");
}

function absolutePath(rootPath, path) {
    return Path.resolve(rootPath, path);
}

function outsideRoot(rootPath, path) {
    return path.indexOf(rootPath) < 0;
}

function outsideRootError(rootPath, paths) {
    var offendingPaths = paths.filter(B.partial(outsideRoot, rootPath));
    if (offendingPaths.length === 0) { return; }
    var plural = offendingPaths.length > 1 ? "Some paths are " : "A path is ";
    var offending = offendingPaths.map(B.partial(Path.relative, rootPath));

    return new Error(plural + "outside the project root. Set rootPath to " +
                     "the desired root to refer to paths outside of " +
                     this.rootPath + ".\n  " + offending.join("\n  "));
}

function resolvePaths(rs, paths, callback) {
    glob(paths, { cwd: rs.rootPath }, function (err, files) {
        var ms = files.map(B.partial(absolutePath, rs.rootPath));
        err = err || outsideRootError(rs.rootPath, ms);
        if (err) { return callback.call(rs, err); }
        ms = ms.map(B.partial(relativePath, rs.rootPath));
        ms = addUnique(ms, rs.matchPaths(paths));
        callback.call(rs, err, ms);
    });
}

function fileReader(fileName) {
    return function () {
        var d = when.defer();
        fs.readFile(fileName, this.encoding, function (err, data) {
            if (err) { return d.resolver.reject(err); }
            d.resolver.resolve(data);
        });
        return d.promise;
    };
}

function prepareResource(rootPath, path, resource) {
    var d = when.defer();
    var fileName = absolutePath(rootPath, path);
    resource.content = fileReader(fileName);
    fileEtag.add(fileName, resource, function (err) {
{'fraction_non_alphanumeric': 0.1535970390932223, 'fraction_numerical': 0.006014341892204488, 'mean_word_length': 3.06390977443609, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1855327', 'n_tokens_mistral': 1370, 'n_tokens_neox': 1348, 'n_words': 328}