<commit_before>var through = require('through')

module.exports = police

function police(options) {
  var exclude = false,
      verify = false,
      excludeLength,
      verifyLength,
      tr = through(write)

  options = options || {}

  if (options.exclude) {
    exclude = true
    excludeLength = options.exclude.length
  }
  if (options.verify) {
    verify = true
    verifyLength = options.verify.length
  }

  return tr

  function write(buf) {
    var str = buf.toString(),
        i

    if (exclude) {
      for (i = 0; i < excludeLength; ++i) {
        if (options.exclude[i].test(str)) return
      }
    }
    if (verify) {
      for (i = 0; i < verifyLength; ++i) {
        if (options.verify[i].test(str)) return this.queue(buf)
      }
      return
    }
    this.queue(buf)
  }
}
<commit_msg>code cleanup<commit_after>var through = require('through')

module.exports = police

function police(_options) {
  var options = _options || {}
    , tr = through(write)
    , exclude = false
    , verify = false
    , exclude_length
    , verify_length


  if (options.exclude) {
    exclude = true
    exclude_length = options.exclude.length
  }
  if (options.verify) {
    verify = true
    verify_length = options.verify.length
  }

  return tr

  function write(buf) {
    var str = buf.toString(),
        i

    if (exclude) {
      for (i = 0; i < exclude_length; ++i) {
        if (options.exclude[i].test(str)) return
      }
    }

    if (verify) {
      for (i = 0; i < verify_length; ++i) {
        if (options.verify[i].test(str)) return tr.queue(buf)
      }
      return
    }

    tr.queue(buf)
  }
}

{'fraction_non_alphanumeric': 0.129210042865891, 'fraction_numerical': 0.002449479485609308, 'mean_word_length': 2.2102161100196462, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3881368', 'n_tokens_mistral': 552, 'n_tokens_neox': 527, 'n_words': 127}