<commit_before>var gulp = require('gulp'),
    concat = require('gulp-concat-sourcemap'),
    uglify = require('gulp-uglify'),
    sass = require('gulp-sass'),
    sourcemaps = require('gulp-sourcemaps'),
    autoprefixer = require('gulp-autoprefixer'),
    base64 = require('gulp-base64'),
    svgmin = require('gulp-svgmin'),
    size = require('gulp-size'),
    Q = require('q'),
    colors = require('colors'),
    argv = require('yargs').argv,
    gulpif = require('gulp-if');

var scripts;
if (argv.dev) {
    scripts = {
        'jquery' : 'node_modules/jquery/dist/jquery.js'
    };
} else {
    scripts = {
        'jquery' : 'node_modules/jquery/dist/jquery.min.js'
    };
}

// paths & options used within the tasks
var paths = {
    scripts : [
        {
            dest : 'html/js',
            destFile : 'public.min.js',
            files :
                [
                    scripts.jquery,
                    'html/js/src/common/svg_icons.js',
                    'html/js/src/public.js'
                ]
        },
        {
            dest : 'html/js',
            destFile : 'admin.min.js',
            files :
                [
                    scripts.jquery,
                    'html/js/src/common/svg_icons.js',
                    'html/js/src/common/md.js',
                    'html/js/src/admin/admin.js',
                    'html/js/src/admin/user.js',
                    'html/js/src/admin.js'
                ]
        }
    ],
    styles : [
        {
            src : 'html/css/sass/*.scss',
            dest : 'html/css',
            options : {
                outputStyle : 'compressed'
            }
        }
    ]
};

// Scripts
gulp.task('scripts', function() {
    var deferred = Q.defer();

    setTimeout(function() {
        var script, uglify_options;
        for (var key in paths.scripts) {
            script = paths.scripts[key];

            uglify_options = {
                mangle : true
            };
            for (var _key in script.uglify_options) {
                uglify_options[_key] = script.uglify_options[_key];
            }

            gulp.src(script.files)
                .pipe(sourcemaps.init())
                .pipe(concat(script.destFile))
                .on('error', function (err) {
                    console.log('   ' + 'Concat JS ERROR'.underline.red);
                    console.log('   ' + err.message.underline.red);
                })
                .pipe(gulpif(!argv.dev, uglify(uglify_options)))
                .on('error', function (err) {
                    console.log('   ' + 'Uglify JS ERROR'.underline.red);
                    console.log('   Line ' + err.lineNumber + ': ' + err.message.underline.red);
                })
                .pipe(sourcemaps.write('.', {
                    includeContent : false,
                    sourceRoot : 'src/'
                }))
                .pipe(size({
                    showFiles: true
                }))
                .pipe(gulp.dest(script.dest));
        }

        deferred.resolve();
    }, 1);

    return deferred.promise;
});

// Styles
gulp.task('styles', function() {
    var deferred = Q.defer();

    setTimeout(function() {
        var style;
        for (var key in paths.styles) {
            style = paths.styles[key];

            gulp.src(style.src)
                .pipe(sourcemaps.init())
                .pipe(sass(style.options))
                .on('error', function (err) {
                    console.log('   ' + 'SASS ERROR'.underline.red);
                    console.log('   ' + err.message.underline.red);
                })
                .pipe(autoprefixer())
                .on('error', function (err) {
                    console.log('   ' + 'Autoprefixer ERROR'.underline.red);
                    console.log('   ' + err.message.underline.red);
                })
                .pipe(sourcemaps.write('.', {
                    includeContent : false,
                    sourceRoot : 'sass/'
                }))
                // inline any files with extensions: svg, png#datauri, or jpg#datauri
                .pipe(base64({
                    baseDir : 'html',
                    extensions : ['svg', /\.png#datauri$/i, /\.jpg#datauri$/i],
                    maxImageSize : 8*1024 // bytes
                }))
                .on('error', function (err) {
                    console.log('   ' + 'Base64 ERROR'.underline.red);
                    console.log('   ' + err.message.underline.red);
                })
                .pipe(size({
                    showFiles: true
                }))
                .pipe(gulp.dest(style.dest));
        }

        deferred.resolve();
    }, 1);

    return deferred.promise;
});

// SVGs
gulp.task('svgs', function() {
    return gulp.src('html/images/**/*.svg')
        .pipe(svgmin())
        .pipe(gulp.dest('html/images'));
});

// Rerun the task when a file changes
gulp.task('watch', function() {
    // watch everything in the JS dirs & sub dirs with extension .js, but exclude the .min.js files
    gulp.watch(['html/js/src/**/*.js'], ['scripts']);
    // watch everything in the sass dirs & sub dirs with extension .scss
    gulp.watch(['html/css/sass/**/*.scss'], ['styles']);
});

// The default task (called when you run `gulp` from cli)
gulp.task('default', ['scripts', 'styles']);<commit_msg>changing config to datauri gen in gulp/CSS<commit_after>var gulp = require('gulp'),
    concat = require('gulp-concat-sourcemap'),
    uglify = require('gulp-uglify'),
    sass = require('gulp-sass'),
    sourcemaps = require('gulp-sourcemaps'),
    autoprefixer = require('gulp-autoprefixer'),
    base64 = require('gulp-base64'),
    svgmin = require('gulp-svgmin'),
    size = require('gulp-size'),
    Q = require('q'),
    colors = require('colors'),
    argv = require('yargs').argv,
    gulpif = require('gulp-if');

var scripts;
if (argv.dev) {
    scripts = {
        'jquery' : 'node_modules/jquery/dist/jquery.js'
    };
} else {
    scripts = {
        'jquery' : 'node_modules/jquery/dist/jquery.min.js'
    };
}

// paths & options used within the tasks
var paths = {
    scripts : [
        {
            dest : 'html/js',
            destFile : 'public.min.js',
            files :
                [
                    scripts.jquery,
                    'html/js/src/common/svg_icons.js',
                    'html/js/src/public.js'
                ]
        },
        {
            dest : 'html/js',
            destFile : 'admin.min.js',
            files :
                [
                    scripts.jquery,
                    'html/js/src/common/svg_icons.js',
                    'html/js/src/common/md.js',
                    'html/js/src/admin/admin.js',
                    'html/js/src/admin/user.js',
                    'html/js/src/admin.js'
                ]
        }
    ],
    styles : [
        {
            src : 'html/css/sass/*.scss',
            dest : 'html/css',
            options : {
                outputStyle : 'compressed'
            }
        }
    ]
};

// Scripts
gulp.task('scripts', function() {
    var deferred = Q.defer();

    setTimeout(function() {
        var script, uglify_options;
        for (var key in paths.scripts) {
            script = paths.scripts[key];

            uglify_options = {
                mangle : true
            };
            for (var _key in script.uglify_options) {
                uglify_options[_key] = script.uglify_options[_key];
            }

            gulp.src(script.files)
                .pipe(sourcemaps.init())
                .pipe(concat(script.destFile))
                .on('error', function (err) {
                    console.log('   ' + 'Concat JS ERROR'.underline.red);
                    console.log('   ' + err.message.underline.red);
                })
                .pipe(gulpif(!argv.dev, uglify(uglify_options)))
                .on('error', function (err) {
                    console.log('   ' + 'Uglify JS ERROR'.underline.red);
                    console.log('   Line ' + err.lineNumber + ': ' + err.message.underline.red);
                })
                .pipe(sourcemaps.write('.', {
                    includeContent : false,
                    sourceRoot : 'src/'
                }))
                .pipe(size({
                    showFiles: true
                }))
                .pipe(gulp.dest(script.dest));
        }

        deferred.resolve();
    }, 1);

    return deferred.promise;
});

// Styles
gulp.task('styles', function() {
    var deferred = Q.defer();

    setTimeout(function() {
        var style;
        for (var key in paths.styles) {
            style = paths.styles[key];

            gulp.src(style.src)
                .pipe(sourcemaps.init())
                .pipe(sass(style.options))
                .on('error', function (err) {
                    console.log('   ' + 'SASS ERROR'.underline.red);
                    console.log('   ' + err.message.underline.red);
                })
                .pipe(autoprefixer())
                .on('error', function (err) {
                    console.log('   ' + 'Autoprefixer ERROR'.underline.red);
                    console.log('   ' + err.message.underline.red);
                })
                .pipe(sourcemaps.write('.', {
                    includeContent : false,
                    sourceRoot : 'sass/'
                }))
                // inline any files with extensions: svg#datauri, png#datauri, or jpg#datauri
                .pipe(base64({
                    baseDir : 'html',
                    extensions : [
                        /\.svg#datauri$/i,
                        /\.png#datauri$/i,
                        /\.jpg#datauri$/i
                    ],
                    maxImageSize : 8*1024 // bytes
                }))
                .on('error', function (err) {
                    console.log('   ' + 'Base64 ERROR'.underline.red);
                    console.log('   ' + err.message.underline.red);
                })
                .pipe(size({
                    showFiles: true
                }))
                .pipe(gulp.dest(style.dest));
        }

        deferred.resolve();
    }, 1);

    return deferred.promise;
});

// SVGs
gulp.task('svgs', function() {
    return gulp.src('html/images/**/*.svg')
        .pipe(svgmin())
        .pipe(gulp.dest('html/images'));
});

// Rerun the task when a file changes
gulp.task('watch', function() {
    // watch everything in the JS dirs & sub dirs with extension .js, but exclude the .min.js files
    gulp.watch(['html/js/src/**/*.js'], ['scripts']);
    // watch everything in the sass dirs & sub dirs with extension .scss
    gulp.watch(['html/css/sass/**/*.scss'], ['styles']);
});

// The default task (called when you run `gulp` from cli)
gulp.task('default', ['scripts', 'styles']);
{'fraction_non_alphanumeric': 0.14743413201138345, 'fraction_numerical': 0.002754062241806665, 'mean_word_length': 1.322814498933902, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '558449', 'n_tokens_mistral': 3261, 'n_tokens_neox': 3064, 'n_words': 555}