<commit_before>import fs from 'fs-extra';
import { includes, isEmpty, compact, forEach, find, eq, chain, reject, isNil, get, isEqual } from 'lodash';
import immutable, { List, Map } from 'immutable';
import webpack from 'webpack';
import chokidar from 'chokidar';
import merge from 'webpack-merge';
import path from 'path';
import updateNotifier from 'update-notifier';
import autoprefixer from 'autoprefixer';
import { cwd, baseDir, packageJson } from './variables';
import spawn from 'cross-spawn';
import gulpLoadPlugins from 'gulp-load-plugins';

const spawnSync = spawn.sync;
const $ = gulpLoadPlugins();

export const isValidOption = (options, targetOption) => includes(options, targetOption);

export const logOptionsError = (options, choice) => {
    genericLog(`Invalid choice: (${choice}). ${!isEmpty(options) ? 'Choose: [' + options.join(', ') + ']' : ''}`, 'red');
};

export const genericLog = (str, color) => {
    $.util.log($.util.colors[color || 'yellow'](str));
};

export const folderAlreadyPresent = (files, folderName) => includes(files, folderName);

export const sanitizePath = (base, path) => {
    let resolvedPath;

    try {
        resolvedPath = path.resolve(base, path);
    } catch (err) {
        genericLog(`${base}/${path} cannot be resolved`, 'red');
        throw new Error(`cannot find ${path} to resolve`);
    }

    return resolvedPath;
};

export const removeLeadingAndTrailingSlashesFromString = (str) => typeof str === 'string' ? str.replace(/^\/|\/$/g, '') : '';

export const sanitizeString = (str) => str.toString().replace(/[ ]*,[ ]*|[ ]+/g, ' ');

export const hasWhiteSpace = (str) => /\s/g.test(str);

export const sortObjByOwnKeys = (obj) => Object.keys(obj).sort().reduce((accObj, key) => {
    accObj[key] = obj[key];
    return accObj;
}, {});

export const getFileIfExists = (modulePath) => {
    let file = null;

    try {
        file = require(`${modulePath}`);
    } catch (err) {
        genericLog(`ERROR: requiring ${path.basename(modulePath)} in ${path.dirname(modulePath)} directory \n ${err}`, 'red');
        throw new Error(err);
    }

    return get(file, 'default', file);
};

export const stylesheetProdRules = (type, regex, customConfig, ExtractTextPlugin) => ({
    test: new RegExp(regex),
    use: ExtractTextPlugin.extract({
        fallback: "style-loader",
        allChunks: true,
        use: compact([
            {
                loader: 'css-loader',
                options: {
                    sourceMap: !!customConfig.devtool
                }
            },
            {
                loader: 'postcss-loader',
                options: {
                    plugins: () => {
                        return [
                            autoprefixer(customConfig.autoprefixerOptions)
                        ];
                    }
                }
            },
            (type !== 'css') ? {
                loader: `${type}-loader`,
                options: {
                    //TODO: update this asap when support is available => !!customConfig.devtool
                    sourceMap: (type !== 'sass') ? !!customConfig.devtool : false
                }
            } : false

        ])
    })
});

export const stylesheetDevRules = (type, regex, customConfig) => ({
    test: new RegExp(regex),
    loaders: compact([
        'style-loader',
        `css-loader${customConfig.devtool ? '?sourceMap' : ''}`,
        {
            loader: 'postcss-loader',
            options: {
                plugins: () => {
                    return [
                        autoprefixer(customConfig.autoprefixerOptions)
                    ];
                }
            }
        },
        (type !== 'css') ? `${type}-loader${customConfig.devtool ? '?sourceMap' : ''}` : false
    ])
});

export const getStats = (NODE_ENV) => ({
    colors: true, hash: true, version: true, timings: true, assets: NODE_ENV === 'production',
    chunks: false, modules: false, reasons: false, children: false, source: false,
    errors: true, errorDetails: true, warnings: true, publicPath: false
});

export const testRequiredFields = (assert, requiredConfigKeys, configKeys, configName) => {
    forEach(requiredConfigKeys, key => {<commit_msg>update sass source maps<commit_after>import fs from 'fs-extra';
import { includes, isEmpty, compact, forEach, find, eq, chain, reject, isNil, get, isEqual } from 'lodash';
import immutable, { List, Map } from 'immutable';
import webpack from 'webpack';
import chokidar from 'chokidar';
import merge from 'webpack-merge';
import path from 'path';
import updateNotifier from 'update-notifier';
import autoprefixer from 'autoprefixer';
import { cwd, baseDir, packageJson } from './variables';
import spawn from 'cross-spawn';
import gulpLoadPlugins from 'gulp-load-plugins';

const spawnSync = spawn.sync;
const $ = gulpLoadPlugins();

export const isValidOption = (options, targetOption) => includes(options, targetOption);

export const logOptionsError = (options, choice) => {
    genericLog(`Invalid choice: (${choice}). ${!isEmpty(options) ? 'Choose: [' + options.join(', ') + ']' : ''}`, 'red');
};

export const genericLog = (str, color) => {
    $.util.log($.util.colors[color || 'yellow'](str));
};

export const folderAlreadyPresent = (files, folderName) => includes(files, folderName);

export const sanitizePath = (base, path) => {
    let resolvedPath;

    try {
        resolvedPath = path.resolve(base, path);
    } catch (err) {
        genericLog(`${base}/${path} cannot be resolved`, 'red');
        throw new Error(`cannot find ${path} to resolve`);
    }

    return resolvedPath;
};

export const removeLeadingAndTrailingSlashesFromString = (str) => typeof str === 'string' ? str.replace(/^\/|\/$/g, '') : '';

export const sanitizeString = (str) => str.toString().replace(/[ ]*,[ ]*|[ ]+/g, ' ');

export const hasWhiteSpace = (str) => /\s/g.test(str);

export const sortObjByOwnKeys = (obj) => Object.keys(obj).sort().reduce((accObj, key) => {
    accObj[key] = obj[key];
    return accObj;
}, {});

export const getFileIfExists = (modulePath) => {
    let file = null;

    try {
        file = require(`${modulePath}`);
    } catch (err) {
        genericLog(`ERROR: requiring ${path.basename(modulePath)} in ${path.dirname(modulePath)} directory \n ${err}`, 'red');
        throw new Error(err);
    }

    return get(file, 'default', file);
};

export const stylesheetProdRules = (type, regex, customConfig, ExtractTextPlugin) => ({
    test: new RegExp(regex),
    use: ExtractTextPlugin.extract({
        fallback: "style-loader",
        allChunks: true,
        use: compact([
            {
                loader: 'css-loader',
                options: {
                    sourceMap: !!customConfig.devtool
                }
            },
            {
                loader: 'postcss-loader',
                options: {
                    sourceMap: !!customConfig.devtool,
                    plugins: () => {
                        return [
                            autoprefixer(customConfig.autoprefixerOptions)
                        ];
                    }
                }
            },
            (type !== 'css') ? {
                loader: `${type}-loader`,
                options: {
                    sourceMap: !!customConfig.devtool
                }
            } : false

        ])
    })
});

export const stylesheetDevRules = (type, regex, customConfig) => ({
    test: new RegExp(regex),
    loaders: compact([
        'style-loader',
        `css-loader${customConfig.devtool ? '?sourceMap' : ''}`,
        {
            loader: 'postcss-loader',
            options: {
                sourceMap: !!customConfig.devtool,
                plugins: () => {
                    return [
                        autoprefixer(customConfig.autoprefixerOptions)
                    ];
                }
            }
        },
        (type !== 'css') ? `${type}-loader${!!customConfig.devtool ? '?sourceMap' : ''}` : false
    ])
});

export const getStats = (NODE_ENV) => ({
    colors: true, hash: true, version: true, timings: true, assets: NODE_ENV === 'production',
    chunks: false, modules: false, reasons: false, children: false, source: false,
    errors: true, errorDetails: true, warnings: true, publicPath: false
});

export const testRequiredFields = (assert, requiredConfigKeys, configKeys, configName) => {
    forEach(requiredConfigKeys, key => {
{'fraction_non_alphanumeric': 0.1510348905972797, 'fraction_numerical': 0.0, 'mean_word_length': 2.370267038660821, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5794907', 'n_tokens_mistral': 2448, 'n_tokens_neox': 2398, 'n_words': 597}