<commit_before>var recurse = require('../');
var test = require('tap').test;
var fs = require('fs');
var mkdirp = require('mkdirp');
var rimraf = require('rimraf');

function createFixtures(prefix) {
  mkdirp.sync(prefix + '-filter/sub');
  mkdirp.sync(prefix + '-filter/sub2');
  fs.openSync(prefix + '-filter/1.txt', 'w');
  fs.openSync(prefix + '-filter/2.tar.gz', 'w');
  fs.openSync(prefix + '-filter/sub/3.txt', 'w');
  fs.openSync(prefix + '-filter/sub2/4.jpg', 'w');
}

test('relname filter', function (t) {
  createFixtures('relname');

  t.plan(2);

  var flat = recurse('relname-filter', function(relname, stat) {
    return !stat.isDirectory() && relname.match(/\.txt$/)
  });
  flat.on('data', function(data) {
    t.ok(data.match(/\d\.txt/));
  });
  flat.on('end', function() {
    rimraf.sync('relname-filter');
  });
});

test('stat filter', function (t) {
  createFixtures('stat');

  t.plan(2);

  var flat = recurse('stat-filter', function(relname, stat) {
    return !stat.isDirectory() && relname.match(/\.txt$/)
  });
  flat.on('data', function(data) {
    t.ok(data.match(/\d\.txt/));
  });
  flat.on('end', function() {
    rimraf.sync('stat-filter');
  });
});
<commit_msg>Cleanup filter tests.<commit_after>var recurse = require('../');
var test = require('tap').test;
var fs = require('fs');
var mkdirp = require('mkdirp');
var rimraf = require('rimraf');

function createFixtures(d) {
  mkdirp.sync(d + '/sub');
  mkdirp.sync(d + '/sub2');
  fs.openSync(d + '/1.txt', 'w');
  fs.openSync(d + '/2.tar.gz', 'w');
  fs.openSync(d + '/sub/3.txt', 'w');
  fs.openSync(d + '/sub2/4.jpg', 'w');
}

test('filter txts', function (t) {
  var d = 'filter-txts';
  createFixtures(d);

  t.plan(3);

  var writes = 0;

  var flat = recurse(d, function(relname, stat) {
    return !stat.isDirectory() && relname.match(/\.txt$/);
  });
  flat.on('data', function(data) {
    t.ok(data.match(/\d\.txt/));
    writes++;
  });
  flat.on('end', function() {
    t.equal(writes, 2);
    rimraf.sync(d);
  });
});

test('filter dirs', function (t) {
  var d = 'filter-dirs';
  createFixtures(d);

  t.plan(3);

  var writes = 0;

  var flat = recurse(d, function(relname, stat) {
    return stat.isDirectory();
  });
  flat.on('data', function(data) {
    t.ok(data.match(new RegExp('^' + d + '/sub\\d?$')));
    writes++;
  });
  flat.on('end', function() {
    t.equal(writes, 2);
    rimraf.sync(d);
  });
});

{'fraction_non_alphanumeric': 0.241635687732342, 'fraction_numerical': 0.008261049153242462, 'mean_word_length': 3.5018587360594795, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '602538', 'n_tokens_mistral': 961, 'n_tokens_neox': 936, 'n_words': 168}