<commit_before>var gulp = require('gulp');
var browserSync = require('browser-sync');
var reload = browserSync.reload;
var historyApiFallback = require('connect-history-api-fallback');
var browserify = require('browserify');
var watchify = require('watchify');
var resolve = require('resolve');
var glob = require('glob').sync;
var source = require('vinyl-source-stream');
var buffer = require('vinyl-buffer');
var rename = require('gulp-rename');
var ghPages = require('gulp-gh-pages');
var concat = require('gulp-concat');
var uglify = require('gulp-uglify');
var minifyCss = require('gulp-minify-css');
var streamqueue = require('streamqueue');

var baseDir = './admin';
var srcDir = baseDir + '/src';
var distDir = baseDir + '/dist';
var stylesDir = srcDir + '/styles';
var scriptsDir = srcDir + '/scripts';

var frontendDependencies = [
  'jquery',
  'bootstrap',
  'superagent',
  'csv'
];


function scriptPipeline(bundle, outfile) {

  /**
   * Run and return the scripts pipeline on bundle
   */

  console.log('Bundling: ' + outfile);

  return bundle
           .pipe(source(outfile))
           .pipe(buffer())
           .pipe(uglify())
           .pipe(gulp.dest(distDir));

}

gulp.task('deploy', function() {

  /**
   * Write the dist files to a gh-pages branch and push to remote
   */

  return gulp.src(distDir + '/*')
    .pipe(ghPages({
      message: "Web App update. " + Date.now()
    }));

});


gulp.task('serve', function() {

  /**
   * Run a reloading server for local development.
   */

  browserSync({
    open: false,
    server: {
        baseDir: distDir
        // use this for apps with client side routing
        // middleware: [historyApiFallback]
    }
  });

});


gulp.task('vendor-scripts', function () {

  /**
   * Provide frontend dependencies as a single bundle.
   */

  var bundler = browserify({});

  frontendDependencies.forEach(function (id) {
    bundler.require(resolve.sync(id), {expose: id});
  });

  return scriptPipeline(bundler.bundle(), 'vendor.min.js');

});


gulp.task('app-scripts', function() {

  /**
   * Provide frontend app as a single bundle.
   */

  var bundler = browserify({
        entries: [
            scriptsDir + '/app.js',
            // glob(scriptsDir + '/components/*.js')
        ],
        // transform: [],
        debug: true,
        cache: {},
        packageCache: {},
        fullPaths: true
  });

  // Don't include vendor dependencies in this bundle
  bundler.external(frontendDependencies);

  var watcher  = watchify(bundler);

  watcher
    .on('update', function() {
      scriptPipeline(watcher.bundle(), 'app.min.js');
    });

  return scriptPipeline(watcher.bundle(), 'app.min.js')
           .pipe(reload({stream: true}));

});


gulp.task('styles', function () {
  /**
   * Provide frontend styles as a single bundle.
   */
  return streamqueue(
    {objectMode: true},
    gulp.src(stylesDir + '/app.css'),
    gulp.src('./node_modules/highlight.js/src/styles/default.css')
  )
    .pipe(concat('app.min.css'))
    .pipe(minifyCss({compatibility: 'ie8'}))
    .pipe(gulp.dest(distDir));
});


gulp.task('default', ['vendor-scripts', 'app-scripts', 'styles', 'serve']);
<commit_msg>Added WARN message<commit_after>var gulp = require('gulp');
var browserSync = require('browser-sync');
var reload = browserSync.reload;
var historyApiFallback = require('connect-history-api-fallback');
var browserify = require('browserify');
var watchify = require('watchify');
var resolve = require('resolve');
var glob = require('glob').sync;
var source = require('vinyl-source-stream');
var buffer = require('vinyl-buffer');
var rename = require('gulp-rename');
var ghPages = require('gulp-gh-pages');
var concat = require('gulp-concat');
var uglify = require('gulp-uglify');
var minifyCss = require('gulp-minify-css');
var streamqueue = require('streamqueue');

var baseDir = './admin';
var srcDir = baseDir + '/src';
var distDir = baseDir + '/dist';
var stylesDir = srcDir + '/styles';
var scriptsDir = srcDir + '/scripts';

var frontendDependencies = [
  'jquery',
  'bootstrap',
  'superagent',
  'csv'
];


function scriptPipeline(bundle, outfile) {

  /**
   * Run and return the scripts pipeline on bundle
   */

  console.log('Bundling: ' + outfile);

  return bundle
           .pipe(source(outfile))
           .pipe(buffer())
           .pipe(uglify())
           .pipe(gulp.dest(distDir));

}

gulp.task('deploy', function() {

  /**
   * Write the dist files to a gh-pages branch and push to remote
   */

  return gulp.src(distDir + '/*')
    .pipe(ghPages({
      message: "Web App update. " + Date.now()
    }));

});


gulp.task('serve', function() {

  /**
   * Run a reloading server for local development.
   */

  browserSync({
    open: false,
    server: {
        baseDir: distDir
        // use this for apps with client side routing
        // middleware: [historyApiFallback]
    }
  });

});


gulp.task('vendor-scripts', function () {

  /**
   * Provide frontend dependencies as a single bundle.
   */

  var bundler = browserify({});

  frontendDependencies.forEach(function (id) {
    bundler.require(resolve.sync(id), {expose: id});
  });

  return scriptPipeline(bundler.bundle(), 'vendor.min.js');

});


gulp.task('app-scripts', function() {

  /**
   * Provide frontend app as a single bundle.
   */

  var bundler = browserify({
        entries: [
            scriptsDir + '/app.js',
            // glob(scriptsDir + '/components/*.js')
        ],
        // transform: [],
        debug: true,
        cache: {},
        packageCache: {},
        fullPaths: true
  });

  // Don't include vendor dependencies in this bundle
  bundler.external(frontendDependencies);

  var watcher  = watchify(bundler);

  watcher
    .on('update', function() {
      scriptPipeline(watcher.bundle(), 'app.min.js');
    });

  return scriptPipeline(watcher.bundle(), 'app.min.js')
           .pipe(reload({stream: true}));

});


gulp.task('styles', function () {
  /**
   * Provide frontend styles as a single bundle.
   */
  return streamqueue(
    {objectMode: true},
    gulp.src(stylesDir + '/app.css'),

    // WARN replace hardcoded value
    gulp.src('./node_modules/highlight.js/src/styles/default.css')
  )
    .pipe(concat('app.min.css'))
    .pipe(minifyCss({compatibility: 'ie8'}))
    .pipe(gulp.dest(distDir));
});


gulp.task('default', ['vendor-scripts', 'app-scripts', 'styles', 'serve']);

{'fraction_non_alphanumeric': 0.16811955168119552, 'fraction_numerical': 0.00031133250311332503, 'mean_word_length': 3.0485192186515437, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6758784', 'n_tokens_mistral': 2079, 'n_tokens_neox': 2006, 'n_words': 464}