<commit_before>'use strict'

const path = require('path')
const fs = require('fs')

const defaults = {
    directory: path.join(__dirname, 'src'),
    excludes: [ 'test' ],
    extentions: {
        stylesheet: '.css',
        script: '.js'
    }
}

const getFiles = (directory, excludes) => {
    let files = []

    const folders = fs.readdirSync(directory)

    folders.forEach(folder => {
        const filePath = path.join(directory, folder)
        const stat = fs.statSync(filePath)

        if (stat && stat.isDirectory()) {
            files = files.concat(getFiles(filePath, excludes))
        } else {
            let keep = true

            excludes.forEach(str => {
                if (filePath.includes(str)) { keep = false }
            })

            if (keep) { files.push(filePath) }
        }
    })

    return files
}

/**
 * Represents a cattleman, who gathers from the modular project directory.
 * @constructor
 * @param {object} options - (optional) Object to override default options.
 */
function Cattleman(options) {
    'use strict'

    if (!(this instanceof Cattleman)) {
        throw new Error('Cattleman needs to be called with the new keyword')
    }

    if (typeof options === 'string') {
        options = {
            directory: options
        }
    }

    this.options = Object.assign(defaults, options)
}

/**
 * Parses project directory and creates a list of valid files.
 * @returns {array} - List of valid files
 */
Cattleman.prototype.gatherFiles = function (extentionFilter) {
    let files = getFiles(this.options.directory, this.options.excludes)

    if (extentionFilter) {
        if (!(typeof extentionFilter === 'string')) { throw new Error('extentionFilter has to be type of string') }
        if (!extentionFilter.startsWith('.')) { throw new Error('extentionFilter has to start with \'.\'') }
<commit_msg>added extentionFilter documentation<commit_after>'use strict'

const path = require('path')
const fs = require('fs')

const defaults = {
    directory: path.join(__dirname, 'src'),
    excludes: [ 'test' ],
    extentions: {
        stylesheet: '.css',
        script: '.js'
    }
}

const getFiles = (directory, excludes) => {
    let files = []

    const folders = fs.readdirSync(directory)

    folders.forEach(folder => {
        const filePath = path.join(directory, folder)
        const stat = fs.statSync(filePath)

        if (stat && stat.isDirectory()) {
            files = files.concat(getFiles(filePath, excludes))
        } else {
            let keep = true

            excludes.forEach(str => {
                if (filePath.includes(str)) { keep = false }
            })

            if (keep) { files.push(filePath) }
        }
    })

    return files
}

/**
 * Represents a cattleman, who gathers from the modular project directory.
 * @constructor
 * @param {object} options - (optional) Object to override default options.
 */
function Cattleman(options) {
    'use strict'

    if (!(this instanceof Cattleman)) {
        throw new Error('Cattleman needs to be called with the new keyword')
    }

    if (typeof options === 'string') {
        options = {
            directory: options
        }
    }

    this.options = Object.assign(defaults, options)
}

/**
 * Parses project directory and creates a list of valid files.
 * @param {string} extentionFilter - (optional) file extention of interest.
 * @returns {array} - List of valid files
 */
Cattleman.prototype.gatherFiles = function (extentionFilter) {
    let files = getFiles(this.options.directory, this.options.excludes)

    if (extentionFilter) {
        if (!(typeof extentionFilter === 'string')) { throw new Error('extentionFilter has to be type of string') }
        if (!extentionFilter.startsWith('.')) { throw new Error('extentionFilter has to start with \'.\'') }

{'fraction_non_alphanumeric': 0.12493465760585468, 'fraction_numerical': 0.0, 'mean_word_length': 2.563314711359404, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2174881', 'n_tokens_mistral': 1093, 'n_tokens_neox': 1064, 'n_words': 316}