<commit_before>'use strict';

const path = require('path');
const gulp = require('gulp');
const rename = require('gulp-rename');
const fsp = require('fs-promise');
const numCPUs = require('os').cpus().length;
const defaultsDeep = require('lodash.defaultsdeep');
const promiseTask = require('./lib/promise-task');
const template = require('./lib/template');
const getNodeModulePath = require('./lib/get-node-module-path');
const getCommitId = require('./lib/get-commit-id');
const VError = require('verror');

const ASSETS_DEFAULT_NAME = './config/assets.default.json';
const ASSETS_DEFAULT_PATH = path.join(__dirname, ASSETS_DEFAULT_NAME);
const GIT_WEBPACK_DEFAULT = require('./config/git-webpack.default.json');
const ASSETS_DEFAULT = require(ASSETS_DEFAULT_NAME);



/**
 * 多进程构建任务管理器
 * @param   {Object}            options
 * @param   {Object[]|string[]} options.modules         模块目录列表
 * @param   {Object}            options.modules.name    模块目录名（相对）
 * @param   {string[]}          options.modules.librarys   模块依赖目录（相对），继承 options.librarys
 * @param   {Object}            options.modules.builder    模块编译器设置，继承 options.builder
 * @param   {string[]}          options.librarys           模块组公共依赖（相对）
 * @param   {string}            options.assets          构建后文件索引表输出路径（相对）
 * @param   {number}            options.parallel        最大进程数
 * @param   {Object}            options.builder           编译器设置
 * @param   {string}            options.builder.name
 * @param   {boolean}           options.builder.force
 * @param   {number}            options.builder.timeout
 * @param   {string}            options.builder.launch                   
 * @param   {string}            options.builder.cwd                   
 * @param   {Object}            options.builder.env                   
 * @param   {string}            options.builder.execPath                   
 * @param   {string}            options.builder.execArgv                   
 * @param   {string}            options.builder.silent             
 * @param   {string[]|number[]} options.builder.stdio           
 * @param   {Object}            options.builder.uid         
 * @param   {string}            options.builder.gid         
 * @param   {string}            context                 工作目录（绝对路径）
 */
module.exports = (options = {}, context = process.cwd()) => {
    options = defaultsDeep({}, options, GIT_WEBPACK_DEFAULT);
    Object.freeze(options);

    let {assets, parallel} = options;
    let modulesChanged = false;
    let assetsPath = path.join(context, assets);
    let preCommit = {};
    let latestCommit = {};

    if (parallel > numCPUs) {
        console.warn(`[warn] 当前计算机 CPU 核心数为 ${numCPUs} 个，parallel 设置为 ${parallel}`);
    }

    return promiseTask.serial([


        // 读取上一次的编译记录
        () => {
            fsp.readFile(assetsPath, 'utf8').catch(() => {
                return new Promise((resolve, reject) => {
                    let basename = path.basename(assetsPath);

                    // 使用 gulp 创建文件可避免目录不存在的问题
                    gulp.src(ASSETS_DEFAULT_PATH)
                        .pipe(rename(basename))
                        .pipe(gulp.dest(path.dirname(assetsPath)))
                        .on('end', errors => {
                            if (errors) {
                                reject(errors);
                            } else {
                                resolve(JSON.stringify(ASSETS_DEFAULT));
                            }
                        });
                });
            }).then(({modules, librarys}) => {
                Object.keys(modules).forEach(name => preCommit[name] = modules[name].commit);
                Object.keys(librarys).forEach(name => preCommit[name] = librarys[name].commit);
            });
        },


        // 转换 modules
        () => {
            return Object.keys(options.modules).map(key => {
                let module = options.modules[key];

                // 转换字符串形式的格式
                if (typeof module === 'string') {
                    module = { name: module, librarys: [], builder: {} };
                }

                // 模块继承父设置
                defaultsDeep(module.librarys, options.librarys);
                defaultsDeep(module.builder, options.builder);

                let builder = module.builder;
                let data = {
                    moduleName: module.name
                };

                // builder 设置变量，路径相关都转成绝对路径
                builder.cwd = path.join(context, template(builder.cwd, data));
                builder.launch = path.join(context, template(builder.launch, data));
                builder.execArgv = builder.execArgv.map(argv => template(argv, data));
                Object.keys(builder.env).forEach(key => builder.env[key] = template(builder.env[key], data));

                return module;
            });
        },


        // 对比版本的修改
        modules => {


            let diff = name => {
                let target = path.join(context, name);
                try {
                    let commit = latestCommit[name] ? latestCommit[name].commit : getCommitId(target);
                    latestCommit[name] = { commit };
                    return commit !== preCommit[name];
                } catch (e) {
                    throw new VError(e, `无法获取变更记录，因为目标不在 git 仓库中 "${target}"`);
                }
            };


            return modules.filter((module) => {

                let moduleChaned = diff(module.name);
                let librarysChanged = module.librarys.filter(name => diff(name)).length;

                if (module.builder.force || librarysChanged || moduleChaned) {
                    Object.freeze(module);
                    modulesChanged = true;
                    return true;
                } else {
                    console.log(`\n[task:ignore] name: ${module.name}\n`);
                    return false;
                }
            });
        },


        // 运行构建器
        modules => {

            let tasks = defaultsDeep(modules).map((module) => {
                let builder = module.builder;
                builder.$builderPath = getNodeModulePath(builder.name, builder.cwd);

                return () => {
                    let builder = require(`./builder/${module.builder.name}`);
                    return builder(module).then(moduleAsset => {
                        console.log(`\n[task:end] name: ${module.name}\n`);
                        return moduleAsset;
                    });
                };
            });
<commit_msg>使用彩色日志<commit_after>'use strict';

const path = require('path');
const gulp = require('gulp');
const rename = require('gulp-rename');
const fsp = require('fs-promise');
const color = require('cli-color');
const numCPUs = require('os').cpus().length;
const defaultsDeep = require('lodash.defaultsdeep');
const promiseTask = require('./lib/promise-task');
const template = require('./lib/template');
const getNodeModulePath = require('./lib/get-node-module-path');
const getCommitId = require('./lib/get-commit-id');
const VError = require('verror');

const ASSETS_DEFAULT_NAME = './config/assets.default.json';
const ASSETS_DEFAULT_PATH = path.join(__dirname, ASSETS_DEFAULT_NAME);
const GIT_WEBPACK_DEFAULT = require('./config/git-webpack.default.json');
const ASSETS_DEFAULT = require(ASSETS_DEFAULT_NAME);



/**
 * 多进程构建任务管理器
 * @param   {Object}            options
 * @param   {Object[]|string[]} options.modules         模块目录列表
 * @param   {Object}            options.modules.name    模块目录名（相对）
 * @param   {string[]}          options.modules.librarys   模块依赖目录（相对），继承 options.librarys
 * @param   {Object}            options.modules.builder    模块编译器设置，继承 options.builder
 * @param   {string[]}          options.librarys           模块组公共依赖（相对）
 * @param   {string}            options.assets          构建后文件索引表输出路径（相对）
 * @param   {number}            options.parallel        最大进程数
 * @param   {Object}            options.builder           编译器设置
 * @param   {string}            options.builder.name
 * @param   {boolean}           options.builder.force
 * @param   {number}            options.builder.timeout
 * @param   {string}            options.builder.launch                   
 * @param   {string}            options.builder.cwd                   
 * @param   {Object}            options.builder.env                   
 * @param   {string}            options.builder.execPath                   
 * @param   {string}            options.builder.execArgv                   
 * @param   {string}            options.builder.silent             
 * @param   {string[]|number[]} options.builder.stdio           
 * @param   {Object}            options.builder.uid         
 * @param   {string}            options.builder.gid         
 * @param   {string}            context                 工作目录（绝对路径）
 */
module.exports = (options = {}, context = process.cwd()) => {
    options = defaultsDeep({}, options, GIT_WEBPACK_DEFAULT);
    Object.freeze(options);

    let {assets, parallel} = options;
    let modulesChanged = false;
    let assetsPath = path.join(context, assets);
    let preCommit = {};
    let latestCommit = {};

    if (parallel > numCPUs) {
        console.warn(color.yellow(`[warn] 当前计算机 CPU 核心数为 ${numCPUs} 个，parallel 设置为 ${parallel}`));
    }

    return promiseTask.serial([


        // 读取上一次的编译记录
        () => {
            fsp.readFile(assetsPath, 'utf8').catch(() => {
                return new Promise((resolve, reject) => {
                    let basename = path.basename(assetsPath);

                    // 使用 gulp 创建文件可避免目录不存在的问题
                    gulp.src(ASSETS_DEFAULT_PATH)
                        .pipe(rename(basename))
                        .pipe(gulp.dest(path.dirname(assetsPath)))
                        .on('end', errors => {
                            if (errors) {
                                reject(errors);
                            } else {
                                resolve(JSON.stringify(ASSETS_DEFAULT));
                            }
                        });
                });
            }).then(({modules, librarys}) => {
                Object.keys(modules).forEach(name => preCommit[name] = modules[name].commit);
                Object.keys(librarys).forEach(name => preCommit[name] = librarys[name].commit);
            });
        },


        // 转换 modules
        () => {
            return Object.keys(options.modules).map(key => {
                let module = options.modules[key];

                // 转换字符串形式的格式
                if (typeof module === 'string') {
                    module = { name: module, librarys: [], builder: {} };
                }

                // 模块继承父设置
                defaultsDeep(module.librarys, options.librarys);
                defaultsDeep(module.builder, options.builder);

                let builder = module.builder;
                let data = {
                    moduleName: module.name
                };

                // builder 设置变量，路径相关都转成绝对路径
                builder.cwd = path.join(context, template(builder.cwd, data));
                builder.launch = path.join(context, template(builder.launch, data));
                builder.execArgv = builder.execArgv.map(argv => template(argv, data));
                Object.keys(builder.env).forEach(key => builder.env[key] = template(builder.env[key], data));

                return module;
            });
        },


        // 对比版本的修改
        modules => {


            let diff = name => {
                let target = path.join(context, name);
                try {
                    let commit = latestCommit[name] ? latestCommit[name].commit : getCommitId(target);
                    latestCommit[name] = { commit };
                    return commit !== preCommit[name];
                } catch (e) {
                    throw new VError(e, `无法获取变更记录，因为目标不在 git 仓库中 "${target}"`);
                }
            };


            return modules.filter((module) => {

                let moduleChaned = diff(module.name);
                let librarysChanged = module.librarys.filter(name => diff(name)).length;

                if (module.builder.force || librarysChanged || moduleChaned) {
                    Object.freeze(module);
                    modulesChanged = true;
                    return true;
                } else {
                    console.log(color.yellow(`\n[task:ignore] name: ${module.name}\n`));
                    return false;
                }
            });
        },


        // 运行构建器
        modules => {

            let tasks = defaultsDeep(modules).map((module) => {
                let builder = module.builder;
                builder.$builderPath = getNodeModulePath(builder.name, builder.cwd);

                return () => {
                    let builder = require(`./builder/${module.builder.name}`);
                    return builder(module).then(moduleAsset => {
                        console.log(`\n[task:end] ${color.green(module.name)}\n`);
                        return moduleAsset;
                    });
                };
            });

{'fraction_non_alphanumeric': 0.12524775118158255, 'fraction_numerical': 0.00015246226558926666, 'mean_word_length': 1.5257989988448208, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '449548', 'n_tokens_mistral': 3713, 'n_tokens_neox': 3672, 'n_words': 638}