<commit_before>util = require 'util'
fs = require 'fs'
path = require 'path'
_ = require 'underscore'
Backbone = require 'backbone'
Queue = require 'queue-async'

resetFiles = (directory, options, callback) ->
  fs.readdir directory, (err, files) ->
    return callback(err) if err
    return callback() unless files

    queue = new Queue(1)

    for file in files
      do (file) -> queue.defer (callback) ->
        pathed_file = path.join(directory, file)
        fs.stat pathed_file, (err, stat) ->
          return callback(err) if err

          return resetFiles(pathed_file, options, callback) if stat.isDirectory() # a directory
          extension = path.extname(pathed_file)
          return callback() unless (extension is '.js' or extension is '.coffee')

          try
            model_type = require(path.join(directory, file))
            return callback() unless (model_type and _.isFunction(model_type) and ((new model_type()) instanceof Backbone.Model) and model_type.resetSchema)
            model_type.resetSchema options, callback

          catch err
            console.log "resetSchemasByDirectory: skipping: #{err}" if options.verbose
            callback()

    queue.await callback

module.exports = class NodeUtils
  @resetSchemasByDirectory: (directory, options, callback) ->
    [options, callback] = [{}, options] if arguments.length is 2

    resetFiles directory, options, (err) ->
      console.log "resetSchemasByDirectory: failed to reset schemas: #{err}" if err
      callback(err)
<commit_msg>find models in dir method<commit_after>util = require 'util'
fs = require 'fs'
path = require 'path'
_ = require 'underscore'
Backbone = require 'backbone'
Queue = require 'queue-async'

module.exports = class NodeUtils

  @findModels = (directory, options, callback) ->

    model_types = []
    findModelsInDirectory = (directory, options, callback) ->
      fs.readdir directory, (err, files) ->

        return callback(err) if err
        return callback(null, model_types) unless files

        queue = new Queue(1)

        for file in files
          do (file) -> queue.defer (callback) ->
            pathed_file = path.join(directory, file)
            fs.stat pathed_file, (err, stat) ->
              return callback(err) if err

              return findModelsInDirectory(pathed_file, options, callback) if stat.isDirectory() # a directory
              extension = path.extname(pathed_file)
              return callback() unless (extension is '.js' or extension is '.coffee')

              try
                model_path = path.join(directory, file)
                model_type = require(model_path)
                return callback() unless (model_type and _.isFunction(model_type) and ((new model_type()) instanceof Backbone.Model) and model_type.resetSchema)
                model_type.path = model_path if options.append_path
                model_types.push(model_type)
                callback()

              catch err
                console.log "resetSchemasByDirectory: skipping: #{err}" if options.verbose
                callback()

        queue.await (err) ->
          callback(err) if err
          callback(null, model_types)

    findModelsInDirectory(directory, options, callback)

  @resetSchemasByDirectory: (directory, options, callback) =>
    [options, callback] = [{}, options] if arguments.length is 2

    @findModels directory, options, (err, model_types) ->

      queue = new Queue(1)

      for model_type in model_types
        do (model_type) -> queue.defer (callback) ->
          model_type.resetSchema options, callback

      queue.await (err) ->
        console.log "resetSchemasByDirectory: failed to reset schemas: #{err}" if err
        callback(err)

{'fraction_non_alphanumeric': 0.09970596097300187, 'fraction_numerical': 0.0013365410318096765, 'mean_word_length': 2.640077821011673, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7181880', 'n_tokens_mistral': 1031, 'n_tokens_neox': 995, 'n_words': 308}