<commit_before>// this keeps a queue of opened file descriptors, and will make
// fs operations wait until some have closed before trying to open more.

var fs = exports = module.exports = {}
fs._originalFs = require("fs")

Object.getOwnPropertyNames(fs._originalFs).forEach(function(prop) {
  var desc = Object.getOwnPropertyDescriptor(fs._originalFs, prop)
  Object.defineProperty(fs, prop, desc)
})

var queue = []
  , constants = require("constants")

fs._curOpen = 0

fs.MIN_MAX_OPEN = 64
fs.MAX_OPEN = 1024

// prevent EMFILE errors
function OpenReq (path, flags, mode, cb) {
  this.path = path
  this.flags = flags
  this.mode = mode
  this.cb = cb
}

function noop () {}

fs.open = gracefulOpen

function gracefulOpen (path, flags, mode, cb) {
  if (typeof mode === "function") cb = mode, mode = null
  if (typeof cb !== "function") cb = noop

  if (fs._curOpen >= fs.MAX_OPEN) {
    queue.push(new OpenReq(path, flags, mode, cb))
    setTimeout(flush)
    return
  }
  open(path, flags, mode, function (er, fd) {
    if (er && er.code === "EMFILE" && fs._curOpen > fs.MIN_MAX_OPEN) {
      // that was too many.  reduce max, get back in queue.
      // this should only happen once in a great while, and only
      // if the ulimit -n is set lower than 1024.
      fs.MAX_OPEN = fs._curOpen - 1
      return fs.open(path, flags, mode, cb)
    }
    cb(er, fd)
  })
}

function open (path, flags, mode, cb) {
  cb = cb || noop
  fs._curOpen ++
  fs._originalFs.open.call(fs, path, flags, mode, function (er, fd) {
    if (er) onclose()
    cb(er, fd)
  })
}

fs.openSync = function (path, flags, mode) {
  var ret
  ret = fs._originalFs.openSync.call(fs, path, flags, mode)
  fs._curOpen ++
  return ret
}

function onclose () {
  fs._curOpen --
  flush()
}

function flush () {
  while (fs._curOpen < fs.MAX_OPEN) {
    var req = queue.shift()
    if (!req) return
    switch (req.constructor.name) {
      case 'OpenReq':
        open(req.path, req.flags || "r", req.mode || 0777, req.cb)
        break
      case 'ReaddirReq':
        readdir(req.path, req.cb)
        break
      default:
        throw new Error('Unknown req type: ' + req.constructor.name)
    }
  }
}

fs.close = function (fd, cb) {
  cb = cb || noop
  fs._originalFs.close.call(fs, fd, function (er) {
    onclose()
    cb(er)
  })
}

fs.closeSync = function (fd) {
  try {
    return fs._originalFs.closeSync.call(fs, fd)
  } finally {
    onclose()
  }
}


// readdir takes a fd as well.
// however, the sync version closes it right away, so
// there's no need to wrap.
// It would be nice to catch when it throws an EMFILE,
// but that's relatively rare anyway.

fs.readdir = gracefulReaddir

function gracefulReaddir (path, cb) {
  if (fs._curOpen >= fs.MAX_OPEN) {
    queue.push(new ReaddirReq(path, cb))
    setTimeout(flush)
    return
  }

  readdir(path, function (er, files) {
    if (er && er.code === "EMFILE" && fs._curOpen > fs.MIN_MAX_OPEN) {
      fs.MAX_OPEN = fs._curOpen - 1
      return fs.readdir(path, cb)
    }
    cb(er, files)
  })
}

function readdir (path, cb) {
  cb = cb || noop
  fs._curOpen ++
  fs._originalFs.readdir.call(fs, path, function (er, files) {
    onclose()
    cb(er, files)
  })
}

function ReaddirReq (path, cb) {
  this.path = path
  this.cb = cb
}


// (re-)implement some things that are known busted or missing.

var constants = require("constants")

// lchmod, broken prior to 0.6.2
// back-port the fix here.<commit_msg>Wrap fs.writeFile.<commit_after>// this keeps a queue of opened file descriptors, and will make
// fs operations wait until some have closed before trying to open more.

var fs = exports = module.exports = {}
fs._originalFs = require("fs")

Object.getOwnPropertyNames(fs._originalFs).forEach(function(prop) {
  var desc = Object.getOwnPropertyDescriptor(fs._originalFs, prop)
  Object.defineProperty(fs, prop, desc)
})

var queue = []
  , constants = require("constants")

fs._curOpen = 0

fs.MIN_MAX_OPEN = 64
fs.MAX_OPEN = 1024

// prevent EMFILE errors
function OpenReq (path, flags, mode, cb) {
  this.path = path
  this.flags = flags
  this.mode = mode
  this.cb = cb
}

function noop () {}

fs.open = gracefulOpen

function gracefulOpen (path, flags, mode, cb) {
  if (typeof mode === "function") cb = mode, mode = null
  if (typeof cb !== "function") cb = noop

  if (fs._curOpen >= fs.MAX_OPEN) {
    queue.push(new OpenReq(path, flags, mode, cb))
    setTimeout(flush)
    return
  }
  open(path, flags, mode, function (er, fd) {
    if (er && er.code === "EMFILE" && fs._curOpen > fs.MIN_MAX_OPEN) {
      // that was too many.  reduce max, get back in queue.
      // this should only happen once in a great while, and only
      // if the ulimit -n is set lower than 1024.
      fs.MAX_OPEN = fs._curOpen - 1
      return fs.open(path, flags, mode, cb)
    }
    cb(er, fd)
  })
}

function open (path, flags, mode, cb) {
  cb = cb || noop
  fs._curOpen ++
  fs._originalFs.open.call(fs, path, flags, mode, function (er, fd) {
    if (er) onclose()
    cb(er, fd)
  })
}

fs.openSync = function (path, flags, mode) {
  var ret
  ret = fs._originalFs.openSync.call(fs, path, flags, mode)
  fs._curOpen ++
  return ret
}

function onclose () {
  fs._curOpen --
  flush()
}

function flush () {
  while (fs._curOpen < fs.MAX_OPEN) {
    var req = queue.shift()
    if (!req) return
    switch (req.constructor.name) {
      case 'OpenReq':
        open(req.path, req.flags || "r", req.mode || 0777, req.cb)
        break
      case 'ReaddirReq':
        readdir(req.path, req.cb)
        break
      case 'WriteFileReq':
        writeFile(req.path, req.data, req.options, req.cb)
        break
      default:
        throw new Error('Unknown req type: ' + req.constructor.name)
    }
  }
}

fs.close = function (fd, cb) {
  cb = cb || noop
  fs._originalFs.close.call(fs, fd, function (er) {
    onclose()
    cb(er)
  })
}

fs.closeSync = function (fd) {
  try {
    return fs._originalFs.closeSync.call(fs, fd)
  } finally {
    onclose()
  }
}


// readdir takes a fd as well.
// however, the sync version closes it right away, so
// there's no need to wrap.
// It would be nice to catch when it throws an EMFILE,
// but that's relatively rare anyway.

fs.readdir = gracefulReaddir

function gracefulReaddir (path, cb) {
  if (fs._curOpen >= fs.MAX_OPEN) {
    queue.push(new ReaddirReq(path, cb))
    setTimeout(flush)
    return
  }

  readdir(path, function (er, files) {
    if (er && er.code === "EMFILE" && fs._curOpen > fs.MIN_MAX_OPEN) {
      fs.MAX_OPEN = fs._curOpen - 1
      return fs.readdir(path, cb)
    }
    cb(er, files)
  })
}

function readdir (path, cb) {
  cb = cb || noop
  fs._curOpen ++
  fs._originalFs.readdir.call(fs, path, function (er, files) {
    onclose()
    cb(er, files)
  })
}

function ReaddirReq (path, cb) {
  this.path = path
  this.cb = cb
}


fs.writeFile = gracefulWriteFile;

function gracefulWriteFile(path, data, options, cb) {
  if (typeof options === "function") cb = options, options = null
  if (typeof cb !== "function") cb = noop

  if (fs._curOpen >= fs.MAX_OPEN) {
    queue.push(new WriteFileReq(path, data, options, cb))
    setTimeout(flush)
    return
  }

  writeFile(path, data, options, function (er) {
    if (er && er.code === "EMFILE" && fs._curOpen > fs.MIN_MAX_OPEN) {
      fs.MAX_OPEN = fs._curOpen - 1
      return fs.writeFile(path, data, options, cb)
    }
    cb(er)
  })
}

function writeFile (path, data, options, cb) {
  cb = cb || noop
  fs._curOpen ++
  fs._originalFs.writeFile.call(fs, path, data, options, function (er) {
    onclose()
    cb(er)
  })
}

function WriteFileReq (path, data, options, cb) {
  this.path = path
  this.data = data
  this.options = options
  this.cb = cb
}


// (re-)implement some things that are known busted or missing.

var constants = require("constants")

// lchmod, broken prior to 0.6.2
// back-port the fix here.
{'fraction_non_alphanumeric': 0.13837194581592607, 'fraction_numerical': 0.005190530446892012, 'mean_word_length': 2.8053949903660884, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3656794', 'n_tokens_mistral': 2840, 'n_tokens_neox': 2755, 'n_words': 812}