<commit_before>const GRAMMAR_OUT = 'grammar.js';
const GRAMMAR_IN = 'grammar.ne';

var fs = require('fs');
var gulp = require('gulp');

var nearley = require('./node_modules/nearley/lib/nearley.js');
var generate = require('./node_modules/nearley/lib/generate.js');
var Compile = require('./node_modules/nearley/lib/compile.js');
var StreamWrapper = require('./node_modules/nearley/lib/stream.js');

gulp.task('compile-grammar', function(cb) {
  // nearley compiling totally not a copy of
  // https://github.com/Hardmath123/nearley/blob/master/bin/nearleyc.js

  var input = fs.createReadStream(GRAMMAR_IN);
  var output = fs.createWriteStream(GRAMMAR_OUT);

  var parserGrammar = new require('./node_modules/nearley/lib/' +
                                  'nearley-language-bootstrapped.js');
  var parser = new nearley.Parser(parserGrammar.ParserRules, parserGrammar.ParserStart);

  var stream = input
    .pipe(new StreamWrapper(parser))
    .on('finish', function() {
      var c = Compile(parser.results[0], {});
      output.write(generate(c, 'grammar'));
      setTimeout(cb, 1000);
    });
});

gulp.task('default', ['compile-grammar'], function() {
  require('./index.js');
});<commit_msg>add `gulp watch`<commit_after>const GRAMMAR_OUT = 'grammar.js';
const GRAMMAR_IN = 'grammar.ne';

var fs = require('fs');
var gulp = require('gulp');

var nearley = require('./node_modules/nearley/lib/nearley.js');
var generate = require('./node_modules/nearley/lib/generate.js');
var Compile = require('./node_modules/nearley/lib/compile.js');
var StreamWrapper = require('./node_modules/nearley/lib/stream.js');

gulp.task('compile-grammar', function(cb) {
  // nearley compiling totally not a copy of
  // https://github.com/Hardmath123/nearley/blob/master/bin/nearleyc.js

  var input = fs.createReadStream(GRAMMAR_IN);
  var output = fs.createWriteStream(GRAMMAR_OUT);

  var parserGrammar = new require('./node_modules/nearley/lib/' +
                                  'nearley-language-bootstrapped.js');
  var parser = new nearley.Parser(parserGrammar.ParserRules, parserGrammar.ParserStart);

  var stream = input
    .pipe(new StreamWrapper(parser))
    .on('finish', function() {
      var c = Compile(parser.results[0], {});
      output.write(generate(c, 'grammar'));
      setTimeout(cb, 1000);
    });
});

gulp.task('default', ['compile-grammar'], function() {
  require('./index.js');
});

gulp.task('watch', ['default'], function() {
  gulp.watch('./*/**/*.tul', ['default']);
});
{'fraction_non_alphanumeric': 0.1752, 'fraction_numerical': 0.0064, 'mean_word_length': 3.9623015873015874, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1138986', 'n_tokens_mistral': 858, 'n_tokens_neox': 807, 'n_words': 143}