<commit_before>var gulp = require('gulp')
var babel = require('gulp-babel')
var babelify = require('babelify');
var watch = require('gulp-watch')
var sourcemaps = require('gulp-sourcemaps')
var buffer = require('vinyl-buffer');
var browserify = require('browserify');
var source = require('vinyl-source-stream');
var livereload = require('gulp-livereload');
var gutil = require('gulp-util');
var open = require('gulp-open');
var clean = require('gulp-clean');
var run = require('gulp-run');

gulp.task('compile', ['clean'], function() {
  return gulp
    .src('src/**/*.js')
    .pipe(sourcemaps.init())
    .pipe(babel({ stage: 0 }))
    .pipe(sourcemaps.write('.'))
    .on("error", function (err) { console.log("Error : " + err.message); this.emit('end') })
    .pipe(gulp.dest('build'))
})

gulp.task('clean', function() {
  return gulp.src('./build', {read: false})
    .pipe(clean());
})

gulp.task('browser-test', ['clean'], function() {
  browserify({
    entries: 'src/test-runner.js',
    debug: true // make sourcemaps work
  })
  .transform(babelify)
  .bundle()
  .on('error', gutil.log.bind(gutil, 'Browserify Error'))
  .pipe(source('bundle.js'))
  .pipe(buffer())
  .pipe(sourcemaps.init({loadMaps: true}))
  .pipe(sourcemaps.write('./'))
  .pipe(gulp.dest('./build/'))
  .pipe(livereload())
})


gulp.task('server', ['compile'], function() {
  return run('node build/server/run.js')
    .exec()
    .pipe(gulp.dest('logs'))
})

gulp.task('watch', function () {
  livereload.listen();
  watch(['src/**/*.js'], function () {
    gulp.start('browser-test')
  });
  gulp.start('browser-test')
  gulp.src('./test-runner.html').pipe(open());
})<commit_msg>output<commit_after>var gulp = require('gulp')
var babel = require('gulp-babel')
var babelify = require('babelify');
var watch = require('gulp-watch')
var sourcemaps = require('gulp-sourcemaps')
var buffer = require('vinyl-buffer');
var browserify = require('browserify');
var source = require('vinyl-source-stream');
var livereload = require('gulp-livereload');
var gutil = require('gulp-util');
var open = require('gulp-open');
var clean = require('gulp-clean');
var run = require('gulp-run');

gulp.task('compile', ['clean'], function() {
  return gulp
    .src('src/**/*.js')
    .pipe(sourcemaps.init())
    .pipe(babel({ stage: 0 }))
    .pipe(sourcemaps.write('.'))
    .on("error", function (err) { console.log("Error : " + err.message); this.emit('end') })
    .pipe(gulp.dest('build'))
})

gulp.task('clean', function() {
  return gulp.src('./build', {read: false})
    .pipe(clean());
})

gulp.task('browser-test', ['clean'], function() {
  browserify({
    entries: 'src/test-runner.js',
    debug: true // make sourcemaps work
  })
  .transform(babelify)
  .bundle()
  .on('error', gutil.log.bind(gutil, 'Browserify Error'))
  .pipe(source('bundle.js'))
  .pipe(buffer())
  .pipe(sourcemaps.init({loadMaps: true}))
  .pipe(sourcemaps.write('./'))
  .pipe(gulp.dest('./build/'))
  .pipe(livereload())
})


gulp.task('server', ['compile'], function() {
  return run('node build/server/run.js')
    .exec()
    .pipe(gulp.dest('output'))
})

gulp.task('watch', function () {
  livereload.listen();
  watch(['src/**/*.js'], function () {
    gulp.start('browser-test')
  });
  gulp.start('browser-test')
  gulp.src('./test-runner.html').pipe(open());
})
{'fraction_non_alphanumeric': 0.23130069089816763, 'fraction_numerical': 0.0006007810153199159, 'mean_word_length': 4.328, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5745491', 'n_tokens_mistral': 1177, 'n_tokens_neox': 1152, 'n_words': 208}