<commit_before>'use strict';

// include gulp and tools
var gulp        = require('gulp');
var $           = require('gulp-load-plugins')();
var exec        = require('child_process').exec;
var browserSync = require('browser-sync');
var reload      = browserSync.reload;
var del         = require('del');
var runSequence = require('run-sequence');
var es          = require('event-stream');
var browserify  = require('browserify');
var source      = require('vinyl-source-stream');
var buffer      = require('vinyl-buffer');
var sourcemaps  = require('gulp-sourcemaps');
var gutil       = require('gulp-util');
var glslify     = require('glslify');
var babelify    = require('babelify');
//var minifyify   = require('minifyify');
//var uglify      = require('gulp-uglify');
var globby      = require('globby');
var argv        = require('yargs').argv;
var karma       = require('karma');
var eslint      = require('gulp-eslint');
var watchify    = require('watchify');
//var rename      = require('gulp-rename');

var target = 'examples/**/*.js';

// Clean output directories
gulp.task('clean', del.bind(null, ['gh-pages', '.tmp']));

// Copy (data) task
// Copy task
gulp.task('copy', function() {
  var dcm = gulp.src(['data/dicom/**/*'])   // dicom data used in demos
    .pipe(gulp.dest('gh-pages/data/dicom'));

  var nii = gulp.src(['data/nifti/**/*'])   // nifti data used in demos
    .pipe(gulp.dest('gh-pages/data/nifti'));

  var nrrd = gulp.src(['data/nrrd/**/*'])   // nifti data used in demos
    .pipe(gulp.dest('gh-pages/data/nrrd'));

  var vtk = gulp.src(['data/vtk/**/*'])   // nifti data used in demos
    .pipe(gulp.dest('gh-pages/data/vtk'));

  var assets = gulp.src(['favicon.ico'])  // fav icon for github page
    .pipe(gulp.dest('gh-pages'));

  return es.merge(dcm, assets)
    .pipe($.size({title: 'copy'}));
});

// HTML task
gulp.task('html', function() {
  return gulp.src([
        '**/*.html',
        '!bower_components{,/**}',
        '!node_modules{,/**}',
        '!test{,/**}',
        '!gh-pages{,/**}',
        '!deprecated{,/**}'
    ])
    .pipe(gulp.dest('gh-pages'))
    .pipe($.size({title: 'html'}));
});

// CSS task
gulp.task('css', function() {
  return gulp.src([
        'examples/**/*.css'
    ])
    .pipe(gulp.dest('gh-pages/examples'))
    .pipe($.size({title: 'css'}));
});

// JS examples task
gulp.task('js-examples', function(cb) {
  // process files of interest
  var keys = Object.keys(argv);
  if(keys.length === 3){
    target = 'examples/' +  Object.keys(argv)[1] + '/*.js';
  }

  globby([target], function(err, files) {
    if (err) {
      cb(err);
    }
    var tasks = files.map(function(entry) {
          // to remove /app layer
          var index = entry.indexOf('/');
          return browserify(
              {entries: [entry],
                debug: true
              })
            .transform(babelify, {
              presets: ['es2015'],
              ignore: ['external/**/*']
              })
            .transform(glslify)
            .bundle()
            .pipe(source(entry.substring(index + 1)))
            .pipe(buffer())
            .pipe(sourcemaps.init({loadMaps: true}))
                .on('error', gutil.log)
            .pipe(sourcemaps.write('./'))
            .pipe(gulp.dest('gh-pages/examples')); 
        });

    // create a merged stream
    es.merge(tasks).on('end', cb);
  });
});

gulp.task('js-examples-watchify', function(cb) {
  // process files of interest
  var keys = Object.keys(argv);
  if(keys.length === 3){
    target = 'examples/' +  Object.keys(argv)[1] + '/*.js';
  }
  globby([target], function(err, files) {
    if (err) {
      cb(err);
    }
    var tasks = files.map(function(entry) {
      // to remove /app layer
      var index = entry.indexOf('/');
          
      var b = browserify(
        {entries: [entry],
          debug: true,
          cache: {},
          packageCache: {},
          plugin: [watchify]
          })
        .transform(babelify, {
          presets: ['es2015'],
          ignore: ['external/**/*']
          })
        .transform(glslify);

      b.on('update', bundle);
      b.on('log', gutil.log);

      function bundle() {
            return b.bundle()
            .pipe(source(entry.substring(index + 1)))
            .pipe(buffer())
            .pipe(sourcemaps.init({loadMaps: true}))
                .on('error', gutil.log)
            .pipe(sourcemaps.write('./'))
            .pipe(gulp.dest('gh-pages/examples'))
            .pipe(reload({stream: true, once: true}));;
          }

      return bundle();
    });
    // create a merged stream
    es.merge(tasks).on('end', cb);
  });
});

gulp.task('build', function(cb) {
  // process files of interest
  globby(['src/ami.js'], function(err, files) {
    if (err) {
      cb(err);
    }

    var tasks = files.map(function(entry) {<commit_msg>cleanup with babel config file<commit_after>'use strict';

// include gulp and tools
var gulp        = require('gulp');
var $           = require('gulp-load-plugins')();
var exec        = require('child_process').exec;
var browserSync = require('browser-sync');
var reload      = browserSync.reload;
var del         = require('del');
var runSequence = require('run-sequence');
var es          = require('event-stream');
var browserify  = require('browserify');
var source      = require('vinyl-source-stream');
var buffer      = require('vinyl-buffer');
var sourcemaps  = require('gulp-sourcemaps');
var gutil       = require('gulp-util');
var glslify     = require('glslify');
var babelify    = require('babelify');
//var minifyify   = require('minifyify');
//var uglify      = require('gulp-uglify');
var globby      = require('globby');
var argv        = require('yargs').argv;
var karma       = require('karma');
var eslint      = require('gulp-eslint');
var watchify    = require('watchify');
//var rename      = require('gulp-rename');

var target = 'examples/**/*.js';

// Clean output directories
gulp.task('clean', del.bind(null, ['gh-pages', '.tmp']));

// Copy (data) task
// Copy task
gulp.task('copy', function() {
  var dcm = gulp.src(['data/dicom/**/*'])   // dicom data used in demos
    .pipe(gulp.dest('gh-pages/data/dicom'));

  var nii = gulp.src(['data/nifti/**/*'])   // nifti data used in demos
    .pipe(gulp.dest('gh-pages/data/nifti'));

  var nrrd = gulp.src(['data/nrrd/**/*'])   // nifti data used in demos
    .pipe(gulp.dest('gh-pages/data/nrrd'));

  var vtk = gulp.src(['data/vtk/**/*'])   // nifti data used in demos
    .pipe(gulp.dest('gh-pages/data/vtk'));

  var assets = gulp.src(['favicon.ico'])  // fav icon for github page
    .pipe(gulp.dest('gh-pages'));

  return es.merge(dcm, assets)
    .pipe($.size({title: 'copy'}));
});

// HTML task
gulp.task('html', function() {
  return gulp.src([
        '**/*.html',
        '!bower_components{,/**}',
        '!node_modules{,/**}',
        '!test{,/**}',
        '!gh-pages{,/**}',
        '!deprecated{,/**}'
    ])
    .pipe(gulp.dest('gh-pages'))
    .pipe($.size({title: 'html'}));
});

// CSS task
gulp.task('css', function() {
  return gulp.src([
        'examples/**/*.css'
    ])
    .pipe(gulp.dest('gh-pages/examples'))
    .pipe($.size({title: 'css'}));
});

// JS examples task
gulp.task('js-examples', function(cb) {
  // process files of interest
  var keys = Object.keys(argv);
  if(keys.length === 3){
    target = 'examples/' +  Object.keys(argv)[1] + '/*.js';
  }

  globby([target], function(err, files) {
    if (err) {
      cb(err);
    }
    var tasks = files.map(function(entry) {
          // to remove /app layer
          var index = entry.indexOf('/');
          return browserify(
              {entries: [entry],
                debug: true
              })
            .transform(babelify)
            .transform(glslify)
            .bundle()
            .pipe(source(entry.substring(index + 1)))
            .pipe(buffer())
            .pipe(sourcemaps.init({loadMaps: true}))
                .on('error', gutil.log)
            .pipe(sourcemaps.write('./'))
            .pipe(gulp.dest('gh-pages/examples')); 
        });

    // create a merged stream
    es.merge(tasks).on('end', cb);
  });
});

gulp.task('js-examples-watchify', function(cb) {
  // process files of interest
  var keys = Object.keys(argv);
  if(keys.length === 3){
    target = 'examples/' +  Object.keys(argv)[1] + '/*.js';
  }
  globby([target], function(err, files) {
    if (err) {
      cb(err);
    }
    var tasks = files.map(function(entry) {
      // to remove /app layer
      var index = entry.indexOf('/');
          
      var b = browserify(
        {entries: [entry],
          debug: true,
          cache: {},
          packageCache: {},
          plugin: [watchify]
          })
        .transform(babelify)
        .transform(glslify);

      b.on('update', bundle);
      b.on('log', gutil.log);

      function bundle() {
            return b.bundle()
            .pipe(source(entry.substring(index + 1)))
            .pipe(buffer())
            .pipe(sourcemaps.init({loadMaps: true}))
                .on('error', gutil.log)
            .pipe(sourcemaps.write('./'))
            .pipe(gulp.dest('gh-pages/examples'))
            .pipe(reload({stream: true, once: true}));;
          }

      return bundle();
    });
    // create a merged stream
    es.merge(tasks).on('end', cb);
  });
});

gulp.task('build', function(cb) {
  // process files of interest
  globby(['src/ami.js'], function(err, files) {
    if (err) {
      cb(err);
    }

    var tasks = files.map(function(entry) {
{'fraction_non_alphanumeric': 0.19709478524401713, 'fraction_numerical': 0.002090082558261051, 'mean_word_length': 2.245167853509664, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4754913', 'n_tokens_mistral': 3188, 'n_tokens_neox': 3123, 'n_words': 636}