<commit_before>var _ = require('lodash')
var disk = require('./disk')

// Returns a lodash chain that calls cb() just after .value()
//
// For example:
// lodashChain(array, cb).method().method().value()
//
// is the same as:
// _.chain(array).method().method().value(); cb()
function lodashChain (array, cb) {
  var chain = _.chain(array)

  function addCallbackOnValue (c) {
    c.value = _.flow(c.value, function (arg) {
      cb()
      return arg
    })
  }

  addCallbackOnValue(chain)

  _.functions(chain)
    .forEach(function (method) {
      chain[method] = _.flow(chain[method], function (arg) {
        var isChain = _.isObject(arg) && arg.__chain__
        if (isChain) addCallbackOnValue(arg)
        return arg
      })
    })

  return chain
}

// Returns a lodash chain that calls .value() and cb()
// automatically after the first .method()
//
// For example:
// lodashChain(array, cb).method()
//
// is the same as:
// _.chain(array).method().value(); cb()
function lowChain (array, cb) {
  var chain = _.chain(array)

  _.functions(chain)
    .forEach(function (method) {
      chain[method] = _.flow(chain[method], function (arg) {
        var res = arg.value ? arg.value() : arg
        cb()
        return res
      })
    })

  return chain
}

function low (file, options) {
  var checksum

  options = _.assign({
    autosave: true,
    async: true
  }, options)

  function save () {
    if (file && options.autosave) {
      var str = low.stringify(db.object)
      if (str === checksum) return
      checksum = str
      options.async ? disk.write(file, str) : disk.writeSync(file, str)
    }
  }

  function db (key) {
    var array
    if (db.object[key]) {
      array = db.object[key]
    } else {
      array = db.object[key] = []
      save()
    }

    var short = lowChain(array, save)
    short.chain = function () {
      return lodashChain(array, save)
    }
    return short
  }

  db.save = function (f) {
    f = f ? f : file
    disk.write(f, low.stringify(db.object))
  }

  db.saveSync = function (f) {
    f = f ? f : file
    disk.writeSync(f, low.stringify(db.object))
  }

  db.object = {}

  if (file) {
    var data = disk.read(file)
    if (data && data.trim() !== '') {
      try {
        db.object = low.parse(data)
      } catch (e) {
        if (e instanceof SyntaxError) e.message = 'Malformed JSON'
        e.message += ' in file:' + file
        throw e
      }
    } else {
      db.saveSync()
    }
  }

  return db
}

low.mixin = function (arg) {
  _.mixin(arg)
}

low.stringify = function (obj) {
  return JSON.stringify(obj, null, 2)
}

low.parse = function (str) {<commit_msg>Refactored the initial load code for better efficiency<commit_after>var _ = require('lodash')
var disk = require('./disk')

// Returns a lodash chain that calls cb() just after .value()
//
// For example:
// lodashChain(array, cb).method().method().value()
//
// is the same as:
// _.chain(array).method().method().value(); cb()
function lodashChain (array, cb) {
  var chain = _.chain(array)

  function addCallbackOnValue (c) {
    c.value = _.flow(c.value, function (arg) {
      cb()
      return arg
    })
  }

  addCallbackOnValue(chain)

  _.functions(chain)
    .forEach(function (method) {
      chain[method] = _.flow(chain[method], function (arg) {
        var isChain = _.isObject(arg) && arg.__chain__
        if (isChain) addCallbackOnValue(arg)
        return arg
      })
    })

  return chain
}

// Returns a lodash chain that calls .value() and cb()
// automatically after the first .method()
//
// For example:
// lodashChain(array, cb).method()
//
// is the same as:
// _.chain(array).method().value(); cb()
function lowChain (array, cb) {
  var chain = _.chain(array)

  _.functions(chain)
    .forEach(function (method) {
      chain[method] = _.flow(chain[method], function (arg) {
        var res = arg.value ? arg.value() : arg
        cb()
        return res
      })
    })

  return chain
}

function low (file, options) {
  var checksum

  options = _.assign({
    autosave: true,
    async: true
  }, options)

  function save () {
    if (file && options.autosave) {
      var str = low.stringify(db.object)
      if (str === checksum) return
      checksum = str
      options.async ? disk.write(file, str) : disk.writeSync(file, str)
    }
  }

  function db (key) {
    var array
    if (db.object[key]) {
      array = db.object[key]
    } else {
      array = db.object[key] = []
      save()
    }

    var short = lowChain(array, save)
    short.chain = function () {
      return lodashChain(array, save)
    }
    return short
  }

  db.save = function (f) {
    f = f ? f : file
    disk.write(f, low.stringify(db.object))
  }

  db.saveSync = function (f) {
    f = f ? f : file
    disk.writeSync(f, low.stringify(db.object))
  }

  db.object = {}

  if (file) {
    var data = (disk.read(file) || '').trim()
    if (data) {
      try {
        db.object = low.parse(data)
      } catch (e) {
        if (e instanceof SyntaxError) e.message = 'Malformed JSON'
        e.message += ' in file:' + file
        throw e
      }
    } else {
      db.saveSync()
    }
  }

  return db
}

low.mixin = function (arg) {
  _.mixin(arg)
}

low.stringify = function (obj) {
  return JSON.stringify(obj, null, 2)
}

low.parse = function (str) {
{'fraction_non_alphanumeric': 0.1543233082706767, 'fraction_numerical': 0.00037593984962406017, 'mean_word_length': 2.313200498132005, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6761153', 'n_tokens_mistral': 1806, 'n_tokens_neox': 1771, 'n_words': 463}