<commit_before>import gulp from 'gulp'
import gutil from 'gulp-util'
import loadPlugins from 'gulp-load-plugins'
import { Instrumenter } from 'isparta'
import del from 'del'
import seq from 'run-sequence'
import yargs from 'yargs'
import globule from 'globule'
import stableStringify from 'json-stable-stringify'
import fsp from 'fs-promise'
import path from 'path'
import marshal from './test/lib/marshal'

const COVERAGE_THRESHOLDS = { global: 80 }
const { CIRCLECI, CIRCLE_TEST_REPORTS, COVERALLS } = process.env

const $ = loadPlugins()
const argv = yargs
  .string('grep')
  .boolean('bail')
  .argv

const runIntegrationTests = () => gulp.src(['test/lib/setup.js', 'test/integration/**/*.js'], { read: false })
  .pipe($.mocha({
    reporter: CIRCLECI ? 'mocha-junit-reporter' : 'spec',
    reporterOptions: CIRCLECI ? {
      mochaFile: `${CIRCLE_TEST_REPORTS}/junit/test-results-${process.version}.xml`
    } : {},
    grep: argv.grep,
    bail: argv.bail
  }))

const buildExpected = async (filePath, expPath, parse) => {
  const data = await fsp.readFile(filePath, 'utf8')
  const parsed = parse(data)
  const marshaled = marshal(parsed)
  const stringified = stableStringify(marshaled, { space: 2 })
  await fsp.outputFile(expPath, stringified, 'utf8')
  gutil.log('Created', gutil.colors.magenta(expPath))
}

gulp.task('clean', () => del('lib'))
<commit_msg>Add mocha compiler option for babel<commit_after>import gulp from 'gulp'
import gutil from 'gulp-util'
import loadPlugins from 'gulp-load-plugins'
import { Instrumenter } from 'isparta'
import del from 'del'
import seq from 'run-sequence'
import yargs from 'yargs'
import globule from 'globule'
import stableStringify from 'json-stable-stringify'
import fsp from 'fs-promise'
import path from 'path'
import marshal from './test/lib/marshal'

const COVERAGE_THRESHOLDS = { global: 80 }
const { CIRCLECI, CIRCLE_TEST_REPORTS, COVERALLS } = process.env

const $ = loadPlugins()
const argv = yargs
  .string('grep')
  .boolean('bail')
  .argv

const runIntegrationTests = () => gulp.src(['test/lib/setup.js', 'test/integration/**/*.js'], { read: false })
  .pipe($.mocha({
    reporter: CIRCLECI ? 'mocha-junit-reporter' : 'spec',
    reporterOptions: CIRCLECI ? {
      mochaFile: `${CIRCLE_TEST_REPORTS}/junit/test-results-${process.version}.xml`
    } : {},
    grep: argv.grep,
    bail: argv.bail,
    compilers: ['js:babel-core/register']
  }))

const buildExpected = async (filePath, expPath, parse) => {
  const data = await fsp.readFile(filePath, 'utf8')
  const parsed = parse(data)
  const marshaled = marshal(parsed)
  const stringified = stableStringify(marshaled, { space: 2 })
  await fsp.outputFile(expPath, stringified, 'utf8')
  gutil.log('Created', gutil.colors.magenta(expPath))
}

gulp.task('clean', () => del('lib'))

{'fraction_non_alphanumeric': 0.14723707664884136, 'fraction_numerical': 0.0035650623885918, 'mean_word_length': 4.589641434262949, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2177102', 'n_tokens_mistral': 1000, 'n_tokens_neox': 984, 'n_words': 235}