<commit_before>var fs = require('fs')

var wx = 'wx'
if (process.version.match(/^v0\.[0-6]/)) {
  var c = require('constants')
  wx = c.O_TRUNC | c.O_CREAT | c.O_WRONLY | c.O_EXCL
}

var os = require('os')
exports.filetime = 'ctime'
if (os.platform() == "win32") {
  exports.filetime = 'mtime'
}

var debug
var util = require('util')
if (util.debuglog)
  debug = util.debuglog('LOCKFILE')
else if (/\blockfile\b/i.test(process.env.NODE_DEBUG))
  debug = function() {
    var msg = util.format.apply(util, arguments)
    console.error('LOCKFILE %d %s', process.pid, msg)
  }
else
  debug = function() {}

var locks = {}

function hasOwnProperty (obj, prop) {
  return Object.prototype.hasOwnProperty.call(obj, prop)
}

process.on('exit', function () {
  debug('exit listener')
  // cleanup
  Object.keys(locks).forEach(exports.unlockSync)
})

// XXX https://github.com/joyent/node/issues/3555
// Remove when node 0.8 is deprecated.
if (/^v0\.[0-8]\./.test(process.version)) {
  debug('uncaughtException, version = %s', process.version)
  process.on('uncaughtException', function H (er) {
    debug('uncaughtException')
    var l = process.listeners('uncaughtException').filter(function (h) {
      return h !== H
    })
    if (!l.length) {
      // cleanup
      try { Object.keys(locks).forEach(exports.unlockSync) } catch (e) {}
      process.removeListener('uncaughtException', H)
      throw er
    }
  })
}

exports.unlock = function (path, cb) {
  debug('unlock', path)
  // best-effort.  unlocking an already-unlocked lock is a noop
  delete locks[path]
  fs.unlink(path, function (unlinkEr) { cb() })
}

exports.unlockSync = function (path) {
  debug('unlockSync', path)
  // best-effort.  unlocking an already-unlocked lock is a noop
  try { fs.unlinkSync(path) } catch (er) {}
  delete locks[path]
}


// if the file can be opened in readonly mode, then it's there.
// if the error is something other than ENOENT, then it's not.
exports.check = function (path, opts, cb) {
  if (typeof opts === 'function') cb = opts, opts = {}
  debug('check', path, opts)
  fs.open(path, 'r', function (er, fd) {
    if (er) {
      if (er.code !== 'ENOENT') return cb(er)
      return cb(null, false)
    }

    if (!opts.stale) {
      return fs.close(fd, function (er) {
        return cb(er, true)
      })
    }

    fs.fstat(fd, function (er, st) {
      if (er) return fs.close(fd, function (er2) {
        return cb(er)
      })

      fs.close(fd, function (er) {
        var age = Date.now() - st[exports.filetime].getTime()
        return cb(er, age <= opts.stale)
      })
    })
  })
}

exports.checkSync = function (path, opts) {
  opts = opts || {}
  debug('checkSync', path, opts)
  if (opts.wait) {
    throw new Error('opts.wait not supported sync for obvious reasons')
  }

  try {
    var fd = fs.openSync(path, 'r')
  } catch (er) {
    if (er.code !== 'ENOENT') throw er
    return false
  }

  if (!opts.stale) {
    try { fs.closeSync(fd) } catch (er) {}
    return true
  }

  // file exists.  however, might be stale
  if (opts.stale) {
    try {
      var st = fs.fstatSync(fd)
    } finally {
      fs.closeSync(fd)
    }
    var age = Date.now() - st[exports.filetime].getTime()
    return (age <= opts.stale)
  }
}



var req = 1
exports.lock = function (path, opts, cb) {
  if (typeof opts === 'function') cb = opts, opts = {}
  opts.req = opts.req || req++
  debug('lock', path, opts)
  opts.start = opts.start || Date.now()

  if (typeof opts.retries === 'number' && opts.retries > 0) {
    debug('has retries', opts.retries)
    cb = (function (orig) { return function cb (er, fd) {
      debug('retry-mutated callback')
      opts.retries -= 1
      if (!er || opts.retries < 0) return orig(er, fd)

      debug('lock retry', path, opts)

      if (opts.retryWait) setTimeout(retry, opts.retryWait)
      else retry()

      function retry () {
        opts.start = Date.now()
        debug('retrying', opts.start)
        exports.lock(path, opts, cb)
      }
    }})(cb)
  }

  // try to engage the lock.
  // if this succeeds, then we're in business.
  fs.open(path, wx, function (er, fd) {
    if (!er) {
      debug('locked', path, fd)
      locks[path] = fd
      return fs.close(fd, function () {
        return cb()
      })
    }
<commit_msg>manage 'retries' so it does not clash with 'wait' polling<commit_after>var fs = require('fs')

var wx = 'wx'
if (process.version.match(/^v0\.[0-6]/)) {
  var c = require('constants')
  wx = c.O_TRUNC | c.O_CREAT | c.O_WRONLY | c.O_EXCL
}

var os = require('os')
exports.filetime = 'ctime'
if (os.platform() == "win32") {
  exports.filetime = 'mtime'
}

var debug
var util = require('util')
if (util.debuglog)
  debug = util.debuglog('LOCKFILE')
else if (/\blockfile\b/i.test(process.env.NODE_DEBUG))
  debug = function() {
    var msg = util.format.apply(util, arguments)
    console.error('LOCKFILE %d %s', process.pid, msg)
  }
else
  debug = function() {}

var locks = {}

function hasOwnProperty (obj, prop) {
  return Object.prototype.hasOwnProperty.call(obj, prop)
}

process.on('exit', function () {
  debug('exit listener')
  // cleanup
  Object.keys(locks).forEach(exports.unlockSync)
})

// XXX https://github.com/joyent/node/issues/3555
// Remove when node 0.8 is deprecated.
if (/^v0\.[0-8]\./.test(process.version)) {
  debug('uncaughtException, version = %s', process.version)
  process.on('uncaughtException', function H (er) {
    debug('uncaughtException')
    var l = process.listeners('uncaughtException').filter(function (h) {
      return h !== H
    })
    if (!l.length) {
      // cleanup
      try { Object.keys(locks).forEach(exports.unlockSync) } catch (e) {}
      process.removeListener('uncaughtException', H)
      throw er
    }
  })
}

exports.unlock = function (path, cb) {
  debug('unlock', path)
  // best-effort.  unlocking an already-unlocked lock is a noop
  delete locks[path]
  fs.unlink(path, function (unlinkEr) { cb() })
}

exports.unlockSync = function (path) {
  debug('unlockSync', path)
  // best-effort.  unlocking an already-unlocked lock is a noop
  try { fs.unlinkSync(path) } catch (er) {}
  delete locks[path]
}


// if the file can be opened in readonly mode, then it's there.
// if the error is something other than ENOENT, then it's not.
exports.check = function (path, opts, cb) {
  if (typeof opts === 'function') cb = opts, opts = {}
  debug('check', path, opts)
  fs.open(path, 'r', function (er, fd) {
    if (er) {
      if (er.code !== 'ENOENT') return cb(er)
      return cb(null, false)
    }

    if (!opts.stale) {
      return fs.close(fd, function (er) {
        return cb(er, true)
      })
    }

    fs.fstat(fd, function (er, st) {
      if (er) return fs.close(fd, function (er2) {
        return cb(er)
      })

      fs.close(fd, function (er) {
        var age = Date.now() - st[exports.filetime].getTime()
        return cb(er, age <= opts.stale)
      })
    })
  })
}

exports.checkSync = function (path, opts) {
  opts = opts || {}
  debug('checkSync', path, opts)
  if (opts.wait) {
    throw new Error('opts.wait not supported sync for obvious reasons')
  }

  try {
    var fd = fs.openSync(path, 'r')
  } catch (er) {
    if (er.code !== 'ENOENT') throw er
    return false
  }

  if (!opts.stale) {
    try { fs.closeSync(fd) } catch (er) {}
    return true
  }

  // file exists.  however, might be stale
  if (opts.stale) {
    try {
      var st = fs.fstatSync(fd)
    } finally {
      fs.closeSync(fd)
    }
    var age = Date.now() - st[exports.filetime].getTime()
    return (age <= opts.stale)
  }
}



var req = 1
exports.lock = function (path, opts, cb) {
  if (typeof opts === 'function') cb = opts, opts = {}
  opts.req = opts.req || req++
  debug('lock', path, opts)
  opts.start = opts.start || Date.now()

  if (typeof opts.retries === 'number' && opts.retries > 0) {
    debug('has retries', opts.retries)
    var retries = opts.retries
    opts.retries = 0
    cb = (function (orig) { return function cb (er, fd) {
      debug('retry-mutated callback')
      retries -= 1
      if (!er || retries < 0) return orig(er, fd)

      debug('lock retry', path, opts)

      if (opts.retryWait) setTimeout(retry, opts.retryWait)
      else retry()

      function retry () {
        opts.start = Date.now()
        debug('retrying', opts.start)
        exports.lock(path, opts, cb)
      }
    }})(cb)
  }

  // try to engage the lock.
  // if this succeeds, then we're in business.
  fs.open(path, wx, function (er, fd) {
    if (!er) {
      debug('locked', path, fd)
      locks[path] = fd
      return fs.close(fd, function () {
        return cb()
      })
    }

{'fraction_non_alphanumeric': 0.16308119361554477, 'fraction_numerical': 0.004510756419153366, 'mean_word_length': 2.6968790081231298, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5161820', 'n_tokens_mistral': 3055, 'n_tokens_neox': 3000, 'n_words': 798}