<commit_before>/**
 * @fileoverview Build file
 * @author nzakas
 */

/* global target */
/* eslint no-use-before-define: "off", no-console: "off" */
"use strict";

//------------------------------------------------------------------------------
// Requirements
//------------------------------------------------------------------------------

require("shelljs/make");

const lodash = require("lodash"),
    checker = require("npm-license"),
    ReleaseOps = require("eslint-release"),
    dateformat = require("dateformat"),
    fs = require("fs"),
    glob = require("glob"),
    markdownlint = require("markdownlint"),
    os = require("os"),
    path = require("path"),
    semver = require("semver"),
    shell = require("shelljs"),
    ejs = require("ejs"),
    loadPerf = require("load-perf"),
    yaml = require("js-yaml");

const cat = shell.cat;
const cd = shell.cd;
const cp = shell.cp;
const echo = shell.echo;
const exec = shell.exec;
const exit = shell.exit;
const find = shell.find;
const ls = shell.ls;
const mkdir = shell.mkdir;
const pwd = shell.pwd;
const rm = shell.rm;
const test = shell.test;

//------------------------------------------------------------------------------
// Settings
//------------------------------------------------------------------------------

/*
 * A little bit fuzzy. My computer has a first CPU speed of 3392 and the perf test
 * always completes in < 3800ms. However, Travis is less predictable due to
 * multiple different VM types. So I'm fudging this for now in the hopes that it
 * at least provides some sort of useful signal.
 */
const PERF_MULTIPLIER = 13e6;

const OPEN_SOURCE_LICENSES = [
    /MIT/, /BSD/, /Apache/, /ISC/, /WTF/, /Public Domain/, /LGPL/
];

//------------------------------------------------------------------------------
// Data
//------------------------------------------------------------------------------

const NODE = "node ", // intentional extra space
    NODE_MODULES = "./node_modules/",
    TEMP_DIR = "./tmp/",
    DEBUG_DIR = "./debug/",
    BUILD_DIR = "./build/",
    DOCS_DIR = "../eslint.github.io/docs",
    SITE_DIR = "../eslint.github.io/",
    PERF_TMP_DIR = path.join(os.tmpdir(), "eslint", "performance"),

    // Utilities - intentional extra space at the end of each string
    MOCHA = `${NODE_MODULES}mocha/bin/_mocha `,
    ESLINT = `${NODE} bin/eslint.js --report-unused-disable-directives `,

    // Files
    MAKEFILE = "./Makefile.js",
    JS_FILES = "\"lib/**/*.js\" \"conf/**/*.js\" \"bin/**/*.js\" \"tools/**/*.js\"",
    JSON_FILES = find("conf/").filter(fileType("json")),
    MARKDOWN_FILES_ARRAY = find("docs/").concat(ls(".")).filter(fileType("md")),
    TEST_FILES = getTestFilePatterns(),
    PERF_ESLINTRC = path.join(PERF_TMP_DIR, "eslintrc.yml"),
    PERF_MULTIFILES_TARGET_DIR = path.join(PERF_TMP_DIR, "eslint"),
    PERF_MULTIFILES_TARGETS = `"${PERF_MULTIFILES_TARGET_DIR + path.sep}{lib,tests${path.sep}lib}${path.sep}**${path.sep}*.js"`,

    // Settings
    MOCHA_TIMEOUT = 10000;

//------------------------------------------------------------------------------
// Helpers
//------------------------------------------------------------------------------

/**
 * Generates file patterns for test files
 * @returns {string} test file patterns
 * @private
 */
function getTestFilePatterns() {
    return ls("tests/lib/").filter(pathToCheck => test("-d", `tests/lib/${pathToCheck}`)).reduce((initialValue, currentValues) => {
        if (currentValues !== "rules") {
            initialValue.push(`"tests/lib/${currentValues}/**/*.js"`);
        }
        return initialValue;
    }, ["\"tests/lib/rules/**/*.js\"", "\"tests/lib/*.js\"", "\"tests/bin/**/*.js\"", "\"tests/tools/**/*.js\""]).join(" ");
}

/**
 * Simple JSON file validation that relies on ES JSON parser.
 * @param {string} filePath Path to JSON.
 * @throws Error If file contents is invalid JSON.
 * @returns {undefined}
 */
function validateJsonFile(filePath) {
    const contents = fs.readFileSync(filePath, "utf8");

    JSON.parse(contents);
}

/**
 * Generates a function that matches files with a particular extension.
 * @param {string} extension The file extension (i.e. "js")
 * @returns {Function} The function to pass into a filter method.
 * @private
 */
function fileType(extension) {
    return function(filename) {
        return filename.slice(filename.lastIndexOf(".") + 1) === extension;
    };
}

/**
 * Generates a static file that includes each rule by name rather than dynamically
 * looking up based on directory. This is used for the browser version of ESLint.
 * @param {string} basedir The directory in which to look for code.
 * @returns {void}
 */
function generateRulesIndex(basedir) {
    let output = "module.exports = function() {\n";

    output += "    var rules = Object.create(null);\n";

    find(`${basedir}rules/`).filter(fileType("js")).forEach(filename => {
        const basename = path.basename(filename, ".js");

        output += `    rules["${basename}"] = require("./rules/${basename}");\n`;
    });

    output += "\n    return rules;\n};";
    output.to(`${basedir}load-rules.js`);
}

/**
 * Executes a command and returns the output instead of printing it to stdout.
 * @param {string} cmd The command string to execute.
 * @returns {string} The result of the executed command.
 */
function execSilent(cmd) {
    return exec(cmd, { silent: true }).stdout;
}

/**
 * Generates a release blog post for eslint.org
 * @param {Object} releaseInfo The release metadata.
 * @returns {void}
 * @private
 */
function generateBlogPost(releaseInfo) {
    const ruleList = ls("lib/rules")

        // Strip the .js extension
        .map(ruleFileName => ruleFileName.replace(/\.js$/, ""))

        /*
         * Sort by length descending. This ensures that rule names which are substrings of other rule names are not
         * matched incorrectly. For example, the string "no-undefined" should get matched with the `no-undefined` rule,
         * instead of getting matched with the `no-undef` rule followed by the string "ined".
         */
        .sort((ruleA, ruleB) => ruleB.length - ruleA.length);
    const output = ejs.render(cat("./templates/blogpost.md.ejs"), Object.assign({ ruleList }, releaseInfo)),
        now = new Date(),
        month = now.getMonth() + 1,
        day = now.getDate(),
        filename = `../eslint.github.io/_posts/${now.getFullYear()}-${
            month < 10 ? `0${month}` : month}-${
            day < 10 ? `0${day}` : day}-eslint-v${
            releaseInfo.version}-released.md`;

    output.to(filename);
}

/**
 * Generates a doc page with formatter result examples
 * @param  {Object} formatterInfo Linting results from each formatter
 * @param  {string} [prereleaseVersion] The version used for a prerelease. This
 *      changes where the output is stored.
 * @returns {void}
 */
function generateFormatterExamples(formatterInfo, prereleaseVersion) {
    const output = ejs.render(cat("./templates/formatter-examples.md.ejs"), formatterInfo);
    let filename = "../eslint.github.io/docs/user-guide/formatters/index.md",
        htmlFilename = "../eslint.github.io/docs/user-guide/formatters/html-formatter-example.html";

    if (prereleaseVersion) {
        filename = filename.replace("/docs", `/docs/${prereleaseVersion}`);
        htmlFilename = htmlFilename.replace("/docs", `/docs/${prereleaseVersion}`);
    }

    output.to(filename);
    formatterInfo.formatterResults.html.result.to(htmlFilename);
}

/**
 * Generate a doc page that lists all of the rules and links to them
 * @param {string} basedir The directory in which to look for code.
 * @returns {void}
 */
function generateRuleIndexPage(basedir) {
    const outputFile = "../eslint.github.io/_data/rules.yml",
        categoryList = "conf/category-list.json",
        categoriesData = JSON.parse(cat(path.resolve(categoryList)));

    find(path.join(basedir, "/lib/rules/")).filter(fileType("js"))
        .map(filename => [filename, path.basename(filename, ".js")])
        .sort((a, b) => a[1].localeCompare(b[1]))
        .forEach(pair => {
            const filename = pair[0];
            const basename = pair[1];
            const rule = require(filename);

            if (rule.meta.deprecated) {
                categoriesData.deprecated.rules.push({
                    name: basename,
                    replacedBy: rule.meta.docs.replacedBy || []
                });
            } else {
                const output = {
                        name: basename,
                        description: rule.meta.docs.description,
                        recommended: rule.meta.docs.recommended || false,
                        fixable: !!rule.meta.fixable
                    },
                    category = lodash.find(categoriesData.categories, { name: rule.meta.docs.category });

                if (!category.rules) {
                    category.rules = [];
                }

                category.rules.push(output);
            }
        });

    const output = yaml.safeDump(categoriesData, { sortKeys: true });

    output.to(outputFile);
}

/**
 * Commits the changes in the site and publishes them to GitHub.
 * @param {string} [tag] The string to tag the commit with.
 * @returns {void}
 */
function publishSite(tag) {
    const currentDir = pwd();

    cd(SITE_DIR);
    exec("git add -A .");
    exec(`git commit -m "Autogenerated new docs and demo at ${dateformat(new Date())}"`);

    if (tag) {
        exec(`git tag ${tag}`);
    }

    exec("git fetch origin && git rebase origin/master");
    exec("git push origin master --tags");
    cd(currentDir);
}

/**
 * Creates a release version tag and pushes to origin.
 * @param {boolean} [ciRelease] Set to true to indicate this is a CI release.
 * @returns {void}
 */
function release(ciRelease) {

    const releaseInfo = ReleaseOps.release(null, ciRelease);

    echo("Generating site");
    target.gensite();
    generateBlogPost(releaseInfo);
    publishSite(`v${releaseInfo.version}`);
    echo("Site has been published");

    echo("Publishing to GitHub");
    ReleaseOps.publishReleaseToGitHub(releaseInfo);
}

/**
 * Creates a prerelease version tag and pushes to origin.
 * @param {string} prereleaseId The prerelease identifier (alpha, beta, etc.)
 * @returns {void}
 */
function prerelease(prereleaseId) {

    const releaseInfo = ReleaseOps.release(prereleaseId);

    echo("Generating site");

    // always write docs into the next major directory (so 2.0.0-alpha.0 writes to 2.0.0)
    target.gensite(semver.inc(releaseInfo.version, "major"));
    generateBlogPost(releaseInfo);
    echo("Site has not been pushed, please update blog post and push manually.");
}


/**
 * Splits a command result to separate lines.
 * @param {string} result The command result string.
 * @returns {array} The separated lines.
 */
function splitCommandResultToLines(result) {
    return result.trim().split("\n");
}
<commit_msg>Build: make prerelease script publish to GitHub/website (#10151)<commit_after>/**
 * @fileoverview Build file
 * @author nzakas
 */

/* global target */
/* eslint no-use-before-define: "off", no-console: "off" */
"use strict";

//------------------------------------------------------------------------------
// Requirements
//------------------------------------------------------------------------------

require("shelljs/make");

const lodash = require("lodash"),
    checker = require("npm-license"),
    ReleaseOps = require("eslint-release"),
    dateformat = require("dateformat"),
    fs = require("fs"),
    glob = require("glob"),
    markdownlint = require("markdownlint"),
    os = require("os"),
    path = require("path"),
    semver = require("semver"),
    shell = require("shelljs"),
    ejs = require("ejs"),
    loadPerf = require("load-perf"),
    yaml = require("js-yaml");

const cat = shell.cat;
const cd = shell.cd;
const cp = shell.cp;
const echo = shell.echo;
const exec = shell.exec;
const exit = shell.exit;
const find = shell.find;
const ls = shell.ls;
const mkdir = shell.mkdir;
const pwd = shell.pwd;
const rm = shell.rm;
const test = shell.test;

//------------------------------------------------------------------------------
// Settings
//------------------------------------------------------------------------------

/*
 * A little bit fuzzy. My computer has a first CPU speed of 3392 and the perf test
 * always completes in < 3800ms. However, Travis is less predictable due to
 * multiple different VM types. So I'm fudging this for now in the hopes that it
 * at least provides some sort of useful signal.
 */
const PERF_MULTIPLIER = 13e6;

const OPEN_SOURCE_LICENSES = [
    /MIT/, /BSD/, /Apache/, /ISC/, /WTF/, /Public Domain/, /LGPL/
];

//------------------------------------------------------------------------------
// Data
//------------------------------------------------------------------------------

const NODE = "node ", // intentional extra space
    NODE_MODULES = "./node_modules/",
    TEMP_DIR = "./tmp/",
    DEBUG_DIR = "./debug/",
    BUILD_DIR = "./build/",
    DOCS_DIR = "../eslint.github.io/docs",
    SITE_DIR = "../eslint.github.io/",
    PERF_TMP_DIR = path.join(os.tmpdir(), "eslint", "performance"),

    // Utilities - intentional extra space at the end of each string
    MOCHA = `${NODE_MODULES}mocha/bin/_mocha `,
    ESLINT = `${NODE} bin/eslint.js --report-unused-disable-directives `,

    // Files
    MAKEFILE = "./Makefile.js",
    JS_FILES = "\"lib/**/*.js\" \"conf/**/*.js\" \"bin/**/*.js\" \"tools/**/*.js\"",
    JSON_FILES = find("conf/").filter(fileType("json")),
    MARKDOWN_FILES_ARRAY = find("docs/").concat(ls(".")).filter(fileType("md")),
    TEST_FILES = getTestFilePatterns(),
    PERF_ESLINTRC = path.join(PERF_TMP_DIR, "eslintrc.yml"),
    PERF_MULTIFILES_TARGET_DIR = path.join(PERF_TMP_DIR, "eslint"),
    PERF_MULTIFILES_TARGETS = `"${PERF_MULTIFILES_TARGET_DIR + path.sep}{lib,tests${path.sep}lib}${path.sep}**${path.sep}*.js"`,

    // Settings
    MOCHA_TIMEOUT = 10000;

//------------------------------------------------------------------------------
// Helpers
//------------------------------------------------------------------------------

/**
 * Generates file patterns for test files
 * @returns {string} test file patterns
 * @private
 */
function getTestFilePatterns() {
    return ls("tests/lib/").filter(pathToCheck => test("-d", `tests/lib/${pathToCheck}`)).reduce((initialValue, currentValues) => {
        if (currentValues !== "rules") {
            initialValue.push(`"tests/lib/${currentValues}/**/*.js"`);
        }
        return initialValue;
    }, ["\"tests/lib/rules/**/*.js\"", "\"tests/lib/*.js\"", "\"tests/bin/**/*.js\"", "\"tests/tools/**/*.js\""]).join(" ");
}

/**
 * Simple JSON file validation that relies on ES JSON parser.
 * @param {string} filePath Path to JSON.
 * @throws Error If file contents is invalid JSON.
 * @returns {undefined}
 */
function validateJsonFile(filePath) {
    const contents = fs.readFileSync(filePath, "utf8");

    JSON.parse(contents);
}

/**
 * Generates a function that matches files with a particular extension.
 * @param {string} extension The file extension (i.e. "js")
 * @returns {Function} The function to pass into a filter method.
 * @private
 */
function fileType(extension) {
    return function(filename) {
        return filename.slice(filename.lastIndexOf(".") + 1) === extension;
    };
}

/**
 * Generates a static file that includes each rule by name rather than dynamically
 * looking up based on directory. This is used for the browser version of ESLint.
 * @param {string} basedir The directory in which to look for code.
 * @returns {void}
 */
function generateRulesIndex(basedir) {
    let output = "module.exports = function() {\n";

    output += "    var rules = Object.create(null);\n";

    find(`${basedir}rules/`).filter(fileType("js")).forEach(filename => {
        const basename = path.basename(filename, ".js");

        output += `    rules["${basename}"] = require("./rules/${basename}");\n`;
    });

    output += "\n    return rules;\n};";
    output.to(`${basedir}load-rules.js`);
}

/**
 * Executes a command and returns the output instead of printing it to stdout.
 * @param {string} cmd The command string to execute.
 * @returns {string} The result of the executed command.
 */
function execSilent(cmd) {
    return exec(cmd, { silent: true }).stdout;
}

/**
 * Generates a release blog post for eslint.org
 * @param {Object} releaseInfo The release metadata.
 * @returns {void}
 * @private
 */
function generateBlogPost(releaseInfo) {
    const ruleList = ls("lib/rules")

        // Strip the .js extension
        .map(ruleFileName => ruleFileName.replace(/\.js$/, ""))

        /*
         * Sort by length descending. This ensures that rule names which are substrings of other rule names are not
         * matched incorrectly. For example, the string "no-undefined" should get matched with the `no-undefined` rule,
         * instead of getting matched with the `no-undef` rule followed by the string "ined".
         */
        .sort((ruleA, ruleB) => ruleB.length - ruleA.length);
    const output = ejs.render(cat("./templates/blogpost.md.ejs"), Object.assign({ ruleList }, releaseInfo)),
        now = new Date(),
        month = now.getMonth() + 1,
        day = now.getDate(),
        filename = `../eslint.github.io/_posts/${now.getFullYear()}-${
            month < 10 ? `0${month}` : month}-${
            day < 10 ? `0${day}` : day}-eslint-v${
            releaseInfo.version}-released.md`;

    output.to(filename);
}

/**
 * Generates a doc page with formatter result examples
 * @param  {Object} formatterInfo Linting results from each formatter
 * @param  {string} [prereleaseVersion] The version used for a prerelease. This
 *      changes where the output is stored.
 * @returns {void}
 */
function generateFormatterExamples(formatterInfo, prereleaseVersion) {
    const output = ejs.render(cat("./templates/formatter-examples.md.ejs"), formatterInfo);
    let filename = "../eslint.github.io/docs/user-guide/formatters/index.md",
        htmlFilename = "../eslint.github.io/docs/user-guide/formatters/html-formatter-example.html";

    if (prereleaseVersion) {
        filename = filename.replace("/docs", `/docs/${prereleaseVersion}`);
        htmlFilename = htmlFilename.replace("/docs", `/docs/${prereleaseVersion}`);
    }

    output.to(filename);
    formatterInfo.formatterResults.html.result.to(htmlFilename);
}

/**
 * Generate a doc page that lists all of the rules and links to them
 * @param {string} basedir The directory in which to look for code.
 * @returns {void}
 */
function generateRuleIndexPage(basedir) {
    const outputFile = "../eslint.github.io/_data/rules.yml",
        categoryList = "conf/category-list.json",
        categoriesData = JSON.parse(cat(path.resolve(categoryList)));

    find(path.join(basedir, "/lib/rules/")).filter(fileType("js"))
        .map(filename => [filename, path.basename(filename, ".js")])
        .sort((a, b) => a[1].localeCompare(b[1]))
        .forEach(pair => {
            const filename = pair[0];
            const basename = pair[1];
            const rule = require(filename);

            if (rule.meta.deprecated) {
                categoriesData.deprecated.rules.push({
                    name: basename,
                    replacedBy: rule.meta.docs.replacedBy || []
                });
            } else {
                const output = {
                        name: basename,
                        description: rule.meta.docs.description,
                        recommended: rule.meta.docs.recommended || false,
                        fixable: !!rule.meta.fixable
                    },
                    category = lodash.find(categoriesData.categories, { name: rule.meta.docs.category });

                if (!category.rules) {
                    category.rules = [];
                }

                category.rules.push(output);
            }
        });

    const output = yaml.safeDump(categoriesData, { sortKeys: true });

    output.to(outputFile);
}

/**
 * Commits the changes in the site and publishes them to GitHub.
 * @param {string} [tag] The string to tag the commit with.
 * @returns {void}
 */
function publishSite(tag) {
    const currentDir = pwd();

    cd(SITE_DIR);
    exec("git add -A .");
    exec(`git commit -m "Autogenerated new docs and demo at ${dateformat(new Date())}"`);

    if (tag) {
        exec(`git tag ${tag}`);
    }

    exec("git fetch origin && git rebase origin/master");
    exec("git push origin master --tags");
    cd(currentDir);
}

/**
 * Creates a release version tag and pushes to origin.
 * @param {boolean} [ciRelease] Set to true to indicate this is a CI release.
 * @returns {void}
 */
function release(ciRelease) {

    const releaseInfo = ReleaseOps.release(null, ciRelease);

    echo("Generating site");
    target.gensite();
    generateBlogPost(releaseInfo);
    publishSite(`v${releaseInfo.version}`);
    echo("Site has been published");

    echo("Publishing to GitHub");
    ReleaseOps.publishReleaseToGitHub(releaseInfo);
}

/**
 * Creates a prerelease version tag and pushes to origin.
 * @param {string} prereleaseId The prerelease identifier (alpha, beta, etc.)
 * @returns {void}
 */
function prerelease(prereleaseId) {

    const releaseInfo = ReleaseOps.release(prereleaseId);

    echo("Generating site");

    // always write docs into the next major directory (so 2.0.0-alpha.0 writes to 2.0.0)
    target.gensite(semver.inc(releaseInfo.version, "major"));
    generateBlogPost(releaseInfo);
    publishSite(`v${releaseInfo.version}`);
    echo("Site has been published");

    echo("Publishing to GitHub");
    ReleaseOps.publishReleaseToGitHub(releaseInfo);
}


/**
 * Splits a command result to separate lines.
 * @param {string} result The command result string.
 * @returns {array} The separated lines.
 */
function splitCommandResultToLines(result) {
    return result.trim().split("\n");
}

{'fraction_non_alphanumeric': 0.20090682384946723, 'fraction_numerical': 0.003491271820448878, 'mean_word_length': 2.354014598540146, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7405910', 'n_tokens_mistral': 6473, 'n_tokens_neox': 6221, 'n_words': 1682}