<commit_before>var concat = require('gulp-concat'),
    eslint = require('gulp-eslint'),
    express = require('express'),
    fs = require('fs'),
    gulp = require('gulp'),
    gulpif = require('gulp-if'),
    gzip = require('gulp-gzip'),
    hb = require('handlebars'),
    less = require('gulp-less'),
    minifyCSS = require('gulp-minify-css'),
    minifyHTML = require('gulp-minify-html'),
    minifyJS = require('gulp-uglify'),
    mocha = require('gulp-mocha'),
    regPartials = require('./lib/gulp-hb-partials.js'),
    zlib = require('zlib');


// Run tests and product coverage
gulp.task('test', ['build'], function () {
    return gulp.src(['test/*.js'])
        .pipe(mocha());
});


// Lint as JS files (including this one)
gulp.task('lint', ['test'], function () {
    return gulp.src([
        'src/js/*.js',
        'gulpfile.js',
        'test/*.js',
        '!node_modules/**'
    ])
    .pipe(eslint({
        rules : {
            'no-mixed-spaces-and-tabs' : 2,
            'space-after-keywords' : 2,
            'semi' : 2,
            'camelcase' : 1,
            'curly' : 2,
            'no-unused-vars' : 0,
            'comma-dangle' : 2,
            'quotes' : 0,
            'indent': 2
        }
    }))
    .pipe(eslint.format());
});


// Minify and combine all CSS
gulp.task('styles', function() {
    return gulp.src([
        'node_modules/bootstrap/dist/css/bootstrap.css',
        'src/less/*.less'
    ])
    .pipe(gulpif(/[.]less$/, less()))
    .pipe(minifyCSS())
    .pipe(concat('all.min.css'))
    .pipe(gzip({ append: false }))
    .pipe(gulp.dest('build/css'));
});


// Minify and combine all JavaScript
gulp.task('scripts', function() {
    return gulp.src([
        'node_modules/jquery/dist/jquery.js',
        'node_modules/bootstrap/dist/js/bootstrap.js',
        'src/js/*.js'
    ])
    .pipe(concat('all.min.js'))
    .pipe(minifyJS({ preserveComments: 'some' }))
    .pipe(gzip({ append: false }))
    .pipe(gulp.dest('build/js'));
});


// Copy static files to build dir
gulp.task('static', function() {
    return gulp.src('src/static/**')
        .pipe(gzip({ append: false }))
        .pipe(gulp.dest('build'));
});


// Register HBpartials
gulp.task('partials', function() {
    return gulp.src('src/views/partials/*.html')
        .pipe(regPartials(hb));
});


// Serve files for local development
gulp.task('serve', function(callback) {
    express()
        .use(function(req, res, next) {
            res.header('Content-Encoding', 'gzip');
            next();
        })
        .use(express.static('build'))
        .use(function(req, res) {
            res.status(404)
                .sendFile(__dirname + '/build/error.html');
        })
        .listen(3000, callback);
});


// Build macro
gulp.task('build', [
    'static',
    'styles',
    'scripts',
    'partials'
]);


// Watch certain files
gulp.task('watch', ['serve', 'lint'], function() {
    gulp.watch(['src/**', 'test/**'], ['lint']);
});


// What to do when you run `$ gulp`
gulp.task('default', ['watch']);<commit_msg>Gulp task to generate posts<commit_after>var concat = require('gulp-concat'),
    eslint = require('gulp-eslint'),
    express = require('express'),
    frontMatter = require('gulp-front-matter'),
    fs = require('fs'),
    gulp = require('gulp'),
    gulpif = require('gulp-if'),
    gzip = require('gulp-gzip'),
    hb = require('handlebars'),
    less = require('gulp-less'),
    marked = require('gulp-marked'),
    minifyCSS = require('gulp-minify-css'),
    minifyHTML = require('gulp-minify-html'),
    minifyJS = require('gulp-uglify'),
    mocha = require('gulp-mocha'),
    regPartials = require('./lib/gulp-hb-partials.js'),
    rename = require('gulp-rename'),
    zlib = require('zlib');


// Run tests and product coverage
gulp.task('test', ['build'], function () {
    return gulp.src(['test/*.js'])
        .pipe(mocha());
});


// Lint as JS files (including this one)
gulp.task('lint', ['test'], function () {
    return gulp.src([
        'src/js/*.js',
        'gulpfile.js',
        'test/*.js',
        '!node_modules/**'
    ])
    .pipe(eslint({
        rules : {
            'no-mixed-spaces-and-tabs' : 2,
            'space-after-keywords' : 2,
            'semi' : 2,
            'camelcase' : 1,
            'curly' : 2,
            'no-unused-vars' : 0,
            'comma-dangle' : 2,
            'quotes' : 0,
            'indent': 2
        }
    }))
    .pipe(eslint.format());
});


// Minify and combine all CSS
gulp.task('styles', function() {
    return gulp.src([
        'node_modules/bootstrap/dist/css/bootstrap.css',
        'src/less/*.less'
    ])
    .pipe(gulpif(/[.]less$/, less()))
    .pipe(minifyCSS())
    .pipe(concat('all.min.css'))
    .pipe(gzip({ append: false }))
    .pipe(gulp.dest('build/css'));
});


// Minify and combine all JavaScript
gulp.task('scripts', function() {
    return gulp.src([
        'node_modules/jquery/dist/jquery.js',
        'node_modules/bootstrap/dist/js/bootstrap.js',
        'src/js/*.js'
    ])
    .pipe(concat('all.min.js'))
    .pipe(minifyJS({ preserveComments: 'some' }))
    .pipe(gzip({ append: false }))
    .pipe(gulp.dest('build/js'));
});


// Copy static files to build dir
gulp.task('static', function() {
    return gulp.src('src/static/**')
        .pipe(gzip({ append: false }))
        .pipe(gulp.dest('build'));
});


// Register HB partials
gulp.task('partials', function() {
    return gulp.src('src/views/partials/*.html')
        .pipe(regPartials(hb));
});


// Generate posts
gulp.task('posts', function() {
    return gulp.src('src/markdown/posts/**/*.md')
        .pipe(frontMatter())
        .pipe(marked())
        .pipe(rename({
            suffix: '/index',
            extname: '.html'
        }))
        .pipe(gulp.dest('build'));
});


// Serve files for local development
gulp.task('serve', function(callback) {
    express()
        .use(function(req, res, next) {
            res.header('Content-Encoding', 'gzip');
            next();
        })
        .use(express.static('build'))
        .use(function(req, res) {
            res.status(404)
                .sendFile(__dirname + '/build/error.html');
        })
        .listen(3000, callback);
});


// Build macro
gulp.task('build', [
    'static',
    'styles',
    'scripts',
    'partials',
    'posts'
]);


// Watch certain files
gulp.task('watch', ['serve', 'lint'], function() {
    gulp.watch(['src/**', 'test/**'], ['lint']);
});


// What to do when you run `$ gulp`
gulp.task('default', ['watch']);
{'fraction_non_alphanumeric': 0.20445529447665547, 'fraction_numerical': 0.004882514494964907, 'mean_word_length': 2.3512269938650308, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5722006', 'n_tokens_mistral': 2278, 'n_tokens_neox': 2205, 'n_words': 399}