<commit_before>'use strict';

var fs = require('fs');
var gulp = require('gulp');
var browserify = require('browserify');
var markdownTOC = require('markdown-toc');

gulp.task('pre-publish', ['build-browser', 'update-readme-toc']);
gulp.task('build-browser', ['build-browser-full', 'build-browser-min']);

gulp.task('build-browser-full', function () {
    return browserify('browser-build.js')
    .bundle()
    .on('error', handleStreamError)
    .pipe(fs.createWriteStream('browser/split-retain.js'))
});

gulp.task('build-browser-min', function () {
    return browserify('browser-build.js')
    .plugin('minifyify', { map: false })
    .bundle()
    .on('error', handleStreamError)
    .pipe(fs.createWriteStream('browser/split-retain.min.js'));
});

gulp.task('update-readme-toc', function () {
    var readme = fs.readFileSync('README.md', { encoding: 'utf-8' });
    readme = markdownTOC.insert(readme);
    fs.writeFileSync('README.md', readme, { encoding: 'utf-8' });
});

function handleStreamError(error) {
    console.error(error.stack);
    this.emit('end');
}<commit_msg>Add update-source-version gulp task<commit_after>'use strict';

var fs = require('fs');
var gulp = require('gulp');
var browserify = require('browserify');
var markdownTOC = require('markdown-toc');
var packageJson = require('./package.json');

gulp.task('pre-publish', ['update-source-version', 'build-browser', 'update-readme-toc']);
gulp.task('build-browser', ['build-browser-full', 'build-browser-min']);

gulp.task('build-browser-full', function () {
    return browserify('browser-build.js')
    .bundle()
    .on('error', handleStreamError)
    .pipe(fs.createWriteStream('browser/split-retain.js'))
});

gulp.task('build-browser-min', function () {
    return browserify('browser-build.js')
    .plugin('minifyify', { map: false })
    .bundle()
    .on('error', handleStreamError)
    .pipe(fs.createWriteStream('browser/split-retain.min.js'));
});

gulp.task('update-readme-toc', function () {
    var readme = fs.readFileSync('README.md', { encoding: 'utf-8' });
    readme = markdownTOC.insert(readme);
    fs.writeFileSync('README.md', readme, { encoding: 'utf-8' });
});

gulp.task('update-source-version', function () {
    var source = fs.readFileSync('split-retain.js', { encoding: 'utf-8' });
    var newSource = source.replace(/(splitRetain\['VERSION'\] = ')[^']+/, '$1' + packageJson.version);

    if (source === newSource) {
        throw new Error('source version change failed: version didn\'t change');
    }

    fs.writeFileSync('split-retain.js', newSource, { encoding: 'utf-8' });
});

function handleStreamError(error) {
    console.error(error.stack);
    this.emit('end');
}
{'fraction_non_alphanumeric': 0.2021553325901152, 'fraction_numerical': 0.002601263470828688, 'mean_word_length': 4.493877551020408, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3247017', 'n_tokens_mistral': 902, 'n_tokens_neox': 889, 'n_words': 147}