<commit_before>'use strict';

const through = require('through2');
const pipeline = require('pipe-iterators').pipeline;

const inputParserStream = require('./streams/input-parser');
const dedupeStream = require('./streams/dedupe');
const handleInputStream = require('./streams/handle-input');

const logStream = require('through2').obj(function (hit, _, cb) {
  console.log(JSON.stringify(hit));
  cb(null, hit);
});

let config = null;

try {
  config = require('./config.json');
} catch (e) {
  console.error('Create config.json (use config.json.base as template) to configure how to handle the input');
}

process.stdin.pipe(
  pipeline(
    inputParserStream(),
    dedupeStream(),
    logStream,
    config ? handleInputStream(config) : through.obj()
  ).on('error', console.error)
);<commit_msg>expect config to be defined<commit_after>'use strict';

const through = require('through2');
const pipeline = require('pipe-iterators').pipeline;

const inputParserStream = require('./streams/input-parser');
const dedupeStream = require('./streams/dedupe');
const handleInputStream = require('./streams/handle-input');

const logStream = through.obj(function (hit, _, cb) {
  console.log(JSON.stringify(hit));
  cb(null, hit);
});

let config = null;

try {
  config = require('./config.json');
} catch (e) {
  console.error('Create config.json (use config.json.base as template) to configure how to handle the input');
}

process.stdin.pipe(
  pipeline(
    inputParserStream(),
    dedupeStream(),
    logStream,
    handleInputStream(config)
  ).on('error', console.error)
);
{'fraction_non_alphanumeric': 0.16022799240025332, 'fraction_numerical': 0.0018999366687777073, 'mean_word_length': 4.851851851851852, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6737225', 'n_tokens_mistral': 473, 'n_tokens_neox': 470, 'n_words': 114}