<commit_before>#!/usr/bin/env node

var es = require('event-stream');
var oj = require('../../lib/boot');

var createProcessor = function(type) {
  return es.map(function(file, cb) {
    var contents = file.contents.toString('utf8');

    contents += type + 'processor has been executed' + "\n";

    file.contents = new Buffer(contents);
    cb(null, file);
  });
};

// pre processors
oj.preProcessor('simplesyntax', function(params) {
  return createProcessor('simple preprocessor that will be overwritten ');
});
oj.preProcessor('simplesyntax', function(params) {
  return createProcessor('simple pre');
});

oj.preProcessor('alternativesyntax', function(params) {
  return [
    createProcessor('alternative pre')
  ];
});

oj.preProcessor('multiplesyntax', function(params) {
  return [
    createProcessor('multiple1 pre'),
    createProcessor('multiple2 pre')
  ];
});

oj.preProcessor('orderedsyntax', function(params) {
  return createProcessor('ordered pre');
});



// post processors
oj.postProcessor('simplesyntax').set(function(params) {
  return createProcessor('simple postprocessor that will be overwritten ');
});
oj.postProcessor('simplesyntax').set(function(params) {
  return createProcessor('simple post');
});
oj.postProcessor('simplesyntax').add(/matching/, function(params) {
  return createProcessor('simple path-matching post');
});

oj.postProcessor('alternativesyntax').add(function(params) {
  return [
    createProcessor('alternative post')
  ];
});

oj.postProcessor('multiplesyntax').add(function(params) {
  return createProcessor('multiple1 post');
});
oj.postProcessor('multiplesyntax').add(function(params) {
  return [
    createProcessor('multiple2 post'),
    createProcessor('multiple3 post')
  ];
});

oj.postProcessor('orderedsyntax').push(function(params) {
  return createProcessor('ordered1 post');
});
oj.postProcessor('orderedsyntax').unshift(function(params) {
  return createProcessor('ordered2 post');
});
oj.postProcessor('orderedsyntax').add(function(params) {
  return [
    createProcessor('ordered3 post'),
    createProcessor('ordered4 post')
  ];
});<commit_msg>Rename old references to OJ<commit_after>#!/usr/bin/env node

var es = require('event-stream');
var lingon = require('../../lib/boot');

var createProcessor = function(type) {
  return es.map(function(file, cb) {
    var contents = file.contents.toString('utf8');

    contents += type + 'processor has been executed' + "\n";

    file.contents = new Buffer(contents);
    cb(null, file);
  });
};

// pre processors
lingon.preProcessor('simplesyntax', function(params) {
  return createProcessor('simple preprocessor that will be overwritten ');
});
lingon.preProcessor('simplesyntax', function(params) {
  return createProcessor('simple pre');
});

lingon.preProcessor('alternativesyntax', function(params) {
  return [
    createProcessor('alternative pre')
  ];
});

lingon.preProcessor('multiplesyntax', function(params) {
  return [
    createProcessor('multiple1 pre'),
    createProcessor('multiple2 pre')
  ];
});

lingon.preProcessor('orderedsyntax', function(params) {
  return createProcessor('ordered pre');
});



// post processors
lingon.postProcessor('simplesyntax').set(function(params) {
  return createProcessor('simple postprocessor that will be overwritten ');
});
lingon.postProcessor('simplesyntax').set(function(params) {
  return createProcessor('simple post');
});
lingon.postProcessor('simplesyntax').add(/matching/, function(params) {
  return createProcessor('simple path-matching post');
});

lingon.postProcessor('alternativesyntax').add(function(params) {
  return [
    createProcessor('alternative post')
  ];
});

lingon.postProcessor('multiplesyntax').add(function(params) {
  return createProcessor('multiple1 post');
});
lingon.postProcessor('multiplesyntax').add(function(params) {
  return [
    createProcessor('multiple2 post'),
    createProcessor('multiple3 post')
  ];
});

lingon.postProcessor('orderedsyntax').push(function(params) {
  return createProcessor('ordered1 post');
});
lingon.postProcessor('orderedsyntax').unshift(function(params) {
  return createProcessor('ordered2 post');
});
lingon.postProcessor('orderedsyntax').add(function(params) {
  return [
    createProcessor('ordered3 post'),
    createProcessor('ordered4 post')
  ];
});
{'fraction_non_alphanumeric': 0.15979620194534508, 'fraction_numerical': 0.004631773969430292, 'mean_word_length': 5.844690966719493, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7627850', 'n_tokens_mistral': 1261, 'n_tokens_neox': 1234, 'n_words': 224}