<commit_before>/**
 * @fileoverview A stream that reads in confidant's configuration files
 *     and writes out ninja build rules.
 */
let Readable = require('stream').Readable;
let Transform = require('stream').Transform;
let debug = require('debug')('confidant/ninja_stream');
let dirname = require('path').dirname;
let flatten = require('lodash/array/flatten');
let glob = require('glob').sync;
let inherits = require('util').inherits;
let readFileSync = require('fs').readFileSync;

function NinjaStream() {
  Transform.call(this, { objectMode: true });
  this.id = 0;

  this.push(`# This ninja build file was generated by confidant.
# Do not modify it directly. Instead, modify one or
# more configure.js files.
`);
}
inherits(NinjaStream, Transform);
module.exports = NinjaStream;

NinjaStream.prototype._transform = function(file, encoding, done) {
  debug(`Parsing ${file}`);
  let build = require(file);
  if (Array.isArray(build)) {
    debug('Build config sync');
    this._syncTransform(file, encoding, build, done);
  } else if (typeof build === 'function') {
    debug('Build config async');
    this._asyncTransform(file, encoding, build, done);
  } else {
    throw new Error(`Invalid build specified in ${file}`);
  }
};

NinjaStream.prototype._syncTransform = function(file, encoding, tasks, done) {
  let contents = readFileSync(file);
  let dir = dirname(file);
  debug(`Checking ${file}... found ${tasks.length} tasks`);
  tasks.forEach(task => {
    let rule = `rule-${this.id++}`;
    let js = task.rule.toString();  // Function.prototype.toString
    let cmd = ninjaEscape(`${contents}(${js})()`.replace(/(\n|\r)/g, ''));
    let inputs = flatten(
      task.inputs.map(input => {
        if (input.includes('*') ||
            input.includes('?') ||
            input.includes('[') ||
            input.includes('!') ||
            input.includes('+') ||
            input.includes('@')) {
          return glob(input, { cwd: dir });
        }

        return input;
      }),
      true /* isDeep */
    )
    .filter(input => input.indexOf(' ') === -1)  // ninja can't handle ws
    .map(input => `${dir}/${ninjaEscape(input)}`);
    let outputs = Array.isArray(task.outputs) ?
      task.outputs.map(output => `${dir}/${ninjaEscape(output)}`).join(' ') :
      `${dir}/${ninjaEscape(task.outputs)}`;

    this.push(`
rule ${rule}
  command = cd ${dir} && node -e "${cmd}"

build ${outputs}: ${rule} ${inputs.join(' ')}
`);
  });

  debug(`Wrote rules from ${file} to build.ninja`);
  done();
};

NinjaStream.prototype._asyncTransform = function(file, encoding, RuleStream, done) {
  let tasks = [];
  let stream = new RuleStream();
  stream.on('data', rule => tasks.push(rule));
  stream.on('end', () => this._syncTransform(file, encoding, tasks, done));
};

function ninjaEscape(str) {
  return str.replace('$', '$$$$');
}<commit_msg>Escape ALL of the dollars signs<commit_after>/**
 * @fileoverview A stream that reads in confidant's configuration files
 *     and writes out ninja build rules.
 */
let Readable = require('stream').Readable;
let Transform = require('stream').Transform;
let debug = require('debug')('confidant/ninja_stream');
let dirname = require('path').dirname;
let flatten = require('lodash/array/flatten');
let glob = require('glob').sync;
let inherits = require('util').inherits;
let readFileSync = require('fs').readFileSync;

function NinjaStream() {
  Transform.call(this, { objectMode: true });
  this.id = 0;

  this.push(`# This ninja build file was generated by confidant.
# Do not modify it directly. Instead, modify one or
# more configure.js files.
`);
}
inherits(NinjaStream, Transform);
module.exports = NinjaStream;

NinjaStream.prototype._transform = function(file, encoding, done) {
  debug(`Parsing ${file}`);
  let build = require(file);
  if (Array.isArray(build)) {
    debug('Build config sync');
    this._syncTransform(file, encoding, build, done);
  } else if (typeof build === 'function') {
    debug('Build config async');
    this._asyncTransform(file, encoding, build, done);
  } else {
    throw new Error(`Invalid build specified in ${file}`);
  }
};

NinjaStream.prototype._syncTransform = function(file, encoding, tasks, done) {
  let contents = readFileSync(file);
  let dir = dirname(file);
  debug(`Checking ${file}... found ${tasks.length} tasks`);
  tasks.forEach(task => {
    let rule = `rule-${this.id++}`;
    let js = task.rule.toString();  // Function.prototype.toString
    let cmd = ninjaEscape(`${contents}(${js})()`.replace(/(\n|\r)/g, ''));
    let inputs = flatten(
      task.inputs.map(input => {
        if (input.includes('*') ||
            input.includes('?') ||
            input.includes('[') ||
            input.includes('!') ||
            input.includes('+') ||
            input.includes('@')) {
          return glob(input, { cwd: dir });
        }

        return input;
      }),
      true /* isDeep */
    )
    .filter(input => input.indexOf(' ') === -1)  // ninja can't handle ws
    .map(input => `${dir}/${ninjaEscape(input)}`);
    let outputs = Array.isArray(task.outputs) ?
      task.outputs.map(output => `${dir}/${ninjaEscape(output)}`).join(' ') :
      `${dir}/${ninjaEscape(task.outputs)}`;

    this.push(`
rule ${rule}
  command = cd ${dir} && node -e "${cmd}"

build ${outputs}: ${rule} ${inputs.join(' ')}
`);
  });

  debug(`Wrote rules from ${file} to build.ninja`);
  done();
};

NinjaStream.prototype._asyncTransform = function(file, encoding, RuleStream, done) {
  let tasks = [];
  let stream = new RuleStream();
  stream.on('data', rule => tasks.push(rule));
  stream.on('end', () => this._syncTransform(file, encoding, tasks, done));
};

function ninjaEscape(str) {
  return str.replace(new RegExp('\\$', 'g'), '$$$$');
}
{'fraction_non_alphanumeric': 0.18779261314374893, 'fraction_numerical': 0.000693601525923357, 'mean_word_length': 3.5923566878980893, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4697270', 'n_tokens_mistral': 1825, 'n_tokens_neox': 1803, 'n_words': 435}