<commit_before>'use strict';

const _ = require('lodash');
const bluebird = require('bluebird');
const negate = require('./functions').negate;
const fsp = require('fs-promise');
const matchOn = require('./files').matchOn;
const joinWith = require('./files').joinWith;
const isDirectory = require('./files').isDirectory;
const sizeMatcher = require('./files').sizeMatcher;
const extMatcher = require('./files').extMatcher;

function flatten(a, b) {
  return a.concat(b);
}

function readFiles(dir) {
  return bluebird.resolve(fsp.readdir(dir));
}

class FileHound {
  constructor() {
    this.searchDirectories = [process.cwd()];
    this.filters = [];
  }

  static create() {
    return new FileHound();
  }

  static any() {
    return bluebird
      .all(arguments)
      .reduce(flatten, []);
  }

  static findFiles(path, globPattern) {
    return FileHound
      .create()
      .paths(path)
      .match(globPattern)
      .find();
  }

  _getFiles(dir) {
    let results = readFiles(dir)
      .map(joinWith(dir));

    this.filters.forEach((filter) => {
      if (this.negateFilters) {
        filter = negate(filter);
      }
      results = results.filter(filter);
    });

    return results;
  }

  addFilter(filter) {
    this.filters.push(filter);
    return this;
  }

  paths() {
    this.searchDirectories = _.uniq(Array.from(arguments));
    return this;
  }

  ext(extension) {
    this.addFilter(extMatcher(extension));
    return this;
  }

  size(bytes) {
    this.addFilter(sizeMatcher(bytes));
    return this;
  }

  isEmpty() {
    this.size(0);
    return this;
  }

  match(globPattern) {
    this.addFilter(matchOn(globPattern));
    return this;
  }

  not() {
    this.negateFilters = true;
    return this;
  }

  search(dir) {
    return this._getFiles(dir)
      .map((file) => {
        return isDirectory(file) ? this.search(file) : file;
      })
      .reduce(flatten, []);
  }

  find() {
    const searches = bluebird.map(this.searchDirectories, (dir) => {
      return this.search(dir);
    });
    return bluebird.all(searches).reduce(flatten);
  }

  getSearchDirectories() {
    return _.cloneDeep(this.searchDirectories);
  }
}

module.exports = FileHound;<commit_msg>Task: Replace lodash uniq with native Set<commit_after>'use strict';

const _ = require('lodash');
const bluebird = require('bluebird');
const negate = require('./functions').negate;
const fsp = require('fs-promise');
const matchOn = require('./files').matchOn;
const joinWith = require('./files').joinWith;
const isDirectory = require('./files').isDirectory;
const sizeMatcher = require('./files').sizeMatcher;
const extMatcher = require('./files').extMatcher;

function flatten(a, b) {
  return a.concat(b);
}

function readFiles(dir) {
  return bluebird.resolve(fsp.readdir(dir));
}

class FileHound {
  constructor() {
    this.searchPaths = new Set();
    this.searchPaths.add(process.cwd());
    this.filters = [];
  }

  static create() {
    return new FileHound();
  }

  static any() {
    return bluebird
      .all(arguments)
      .reduce(flatten, []);
  }

  static findFiles(path, globPattern) {
    return FileHound
      .create()
      .paths(path)
      .match(globPattern)
      .find();
  }

  _getFiles(dir) {
    let results = readFiles(dir)
      .map(joinWith(dir));

    this.filters.forEach((filter) => {
      if (this.negateFilters) {
        filter = negate(filter);
      }
      results = results.filter(filter);
    });

    return results;
  }

  addFilter(filter) {
    this.filters.push(filter);
    return this;
  }

  paths() {
    this.searchPaths = new Set(arguments);
    return this;
  }

  ext(extension) {
    this.addFilter(extMatcher(extension));
    return this;
  }

  size(bytes) {
    this.addFilter(sizeMatcher(bytes));
    return this;
  }

  isEmpty() {
    this.size(0);
    return this;
  }

  match(globPattern) {
    this.addFilter(matchOn(globPattern));
    return this;
  }

  not() {
    this.negateFilters = true;
    return this;
  }

  search(dir) {
    return this._getFiles(dir)
      .map((file) => {
        return isDirectory(file) ? this.search(file) : file;
      })
      .reduce(flatten, []);
  }

  find() {
    const searches = bluebird.map(this.searchPaths, (dir) => {
      return this.search(dir);
    });
    return bluebird.all(searches).reduce(flatten);
  }

  getSearchDirectories() {
    const copy = [];
    for (const i of this.searchPaths.values()) {
      copy.push(i);
    }
    return copy;
  }
}

module.exports = FileHound;
{'fraction_non_alphanumeric': 0.15374420145791914, 'fraction_numerical': 0.0004417936823503424, 'mean_word_length': 2.9203463203463205, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3777942', 'n_tokens_mistral': 1511, 'n_tokens_neox': 1474, 'n_words': 276}