<commit_before>var es = require('event-stream');
var Meta = require('meta-script')();
var gutil = require('gulp-util');
var Buffer = require('buffer').Buffer;
var path = require('path');

function streamify(obj) {
  return require('streamifier').createReadStream(obj);
}

function compileFile(file, done) {

  if (file.isNull())
    return done(null, file);

  var stream = file.isStream() ? file.contents : streamify(file.contents);
  var compiler = Meta.createCompiler({name: file.name});
  return es.pipeline(
    stream,
    es.split(),
    es.through(
      function processLine(line) {
        try {
            compiler.parser.processLine(line);
        } catch (e) {
            done(e, null);
        }
      },
      function end() {
        try {
          compiler.parseDone();
          var ast = compiler.produceAst();
          var result = compiler.generate(ast);
          var jsFile = new gutil.File({
              cwd: file.cwd,
              base: file.base,
              path: gutil.replaceExtension(file.path, '.js'),
              contents: new Buffer(result.code)
          });
          done(null, jsFile);
        } catch (e) {
          done(e, null);
        }
      }));
}

module.exports = function () {
  return es.map(compileFile);
};<commit_msg>use createLineStreamCompiler<commit_after>var es = require('event-stream');
var Meta = require('meta-script')();
var gutil = require('gulp-util');
var Buffer = require('buffer').Buffer;
var path = require('path');

function streamify(obj) {
  return require('streamifier').createReadStream(obj);
}

function compileFile(file, done) {

  if (file.isNull())
    return done(null, file);

  var stream = file.isStream() ? file.contents : streamify(file.contents);
  var compiler = Meta.createLineStreamCompiler(file.name);
  return es.pipeline(
    stream,
    es.split(),
    es.through(
      function processLine(line) {
        try {
            compiler.onLine(line);
        } catch (e) {
            done(e, null);
        }
      },
      function end() {
        try {
          var result = compiler.done();
          var jsFile = new gutil.File({
              cwd: file.cwd,
              base: file.base,
              path: gutil.replaceExtension(file.path, '.js'),
              contents: new Buffer(result.code)
          });
          done(null, jsFile);
        } catch (e) {
          done(e, null);
        }
      }));
}

module.exports = function () {
  return es.map(compileFile);
};
{'fraction_non_alphanumeric': 0.14538864277084174, 'fraction_numerical': 0.0, 'mean_word_length': 2.2301690507152148, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1574921', 'n_tokens_mistral': 734, 'n_tokens_neox': 732, 'n_words': 162}