<commit_before>'use strict'
const Bluebird = require('bluebird')
const Buffer = require('safe-buffer').Buffer
const crypto = require('crypto')
const mkdirpCB = require('mkdirp')
const os = require('os')
const path = require('path')
const url = require('url')
const inflight = require('promise-inflight')

const fs = use('fs-promises')
const mkdirp = use('mkdirp')
const promisify = use('promisify')
const zlib = use('zlib-promises')

const pathDirname = promisify.args(path.dirname)

exports.readFile = readFile
exports.clearFile = clearFile
exports.readUrl = readUrl
exports.clearUrl = clearUrl
exports.invalidateUrl = invalidateUrl

const invalidated = {}

function resolveCall () {
  return Bluebird.all(arguments).then(args => {
    const fn = args.shift()
    return Bluebird.resolve(fn.apply(null, args))
  })
}

function cacheFilename (filename) {
  return Bluebird.resolve(filename).then(filename => {
    return path.join(os.homedir(), '.fetch-fic', filename)
  })
}

function readFile (filename, onMiss) {
  const cacheFile = cacheFilename(filename)
  return inflight(['read:', filename], thenReadFile)

  function thenReadFile () {
    return fs.readFile(cacheFile).catch(elseHandleMiss)
  }
  function elseHandleMiss () {
    return resolveCall(onMiss).then(content => writeFile(filename, Buffer.from(content)))
  }
}

function writeFile (filename, content) {
  const cacheFile = cacheFilename(filename)
  return inflight(['write:', filename], thenWriteFile).then(() => content)

  function thenWriteFile () {
    return mkdirp(pathDirname(cacheFile)).then(() => fs.writeFile(cacheFile, content))
  }
}

function clearFile (filename) {
  const cacheFile = cacheFilename(filename)
  return ignoreHarmlessErrors(fs.unlink(cacheFile))
}

function readJSON (filename, onMiss) {
  return readFile(filename, stringifyOnMiss).then(result => JSON.parse(result))
  function stringifyOnMiss () {
    return resolveCall(onMiss).then(result => JSON.stringify(result, null, 2))
  }
}

/*
function writeJSON (filename, content) {
  return writeFile(filename, JSON.stringify(content, null, 2))
}
*/

function readGzipFile (filename, onMiss) {
  return readFile(filename, gzipOnMiss).then(buf => zlib.gunzip(buf).catch(() => {
    return clearFile(filename).then(() => readGzipFile(filename, onMiss))
  }))

  function gzipOnMiss () {
    return resolveCall(onMiss).then(result => zlib.gzip(result))
  }
}

function writeGzipFile (filename, content) {
  return writeFile(filename, zlib.gzip(content)).then(() => content)
}

function getUrlHash (toFetch) {
  return Bluebird.resolve(toFetch).then(toFetch => {
    const parsed = url.parse(toFetch)<commit_msg>cache: handle corrupted JSON correctly<commit_after>'use strict'
const Bluebird = require('bluebird')
const Buffer = require('safe-buffer').Buffer
const crypto = require('crypto')
const mkdirpCB = require('mkdirp')
const os = require('os')
const path = require('path')
const url = require('url')
const inflight = require('promise-inflight')

const fs = use('fs-promises')
const mkdirp = use('mkdirp')
const promisify = use('promisify')
const zlib = use('zlib-promises')

const pathDirname = promisify.args(path.dirname)

exports.readFile = readFile
exports.clearFile = clearFile
exports.readUrl = readUrl
exports.clearUrl = clearUrl
exports.invalidateUrl = invalidateUrl

const invalidated = {}

function resolveCall () {
  return Bluebird.all(arguments).then(args => {
    const fn = args.shift()
    return Bluebird.resolve(fn.apply(null, args))
  })
}

function cacheFilename (filename) {
  return Bluebird.resolve(filename).then(filename => {
    return path.join(os.homedir(), '.fetch-fic', filename)
  })
}

function readFile (filename, onMiss) {
  const cacheFile = cacheFilename(filename)
  return inflight(['read:', filename], thenReadFile)

  function thenReadFile () {
    return fs.readFile(cacheFile).catch(elseHandleMiss)
  }
  function elseHandleMiss () {
    return resolveCall(onMiss).then(content => writeFile(filename, Buffer.from(content)))
  }
}

function writeFile (filename, content) {
  const cacheFile = cacheFilename(filename)
  return inflight(['write:', filename], thenWriteFile).then(() => content)

  function thenWriteFile () {
    return mkdirp(pathDirname(cacheFile)).then(() => fs.writeFile(cacheFile, content))
  }
}

function clearFile (filename) {
  const cacheFile = cacheFilename(filename)
  return ignoreHarmlessErrors(fs.unlink(cacheFile))
}

async function readJSON (filename, onMiss) {
  let didMiss = false
  const result = await readFile(filename, stringifyOnMiss)
  try {
    return JSON.parse(result)
  } catch (ex) {
    if (didMiss) throw ex
    await clearFile(filename)
    return JSON.parse(await readFile(filename, stringifyOnMiss))
  }

  function stringifyOnMiss () {
    didMiss = true
    return resolveCall(onMiss).then(result => JSON.stringify(result, null, 2))
  }
}

/*
function writeJSON (filename, content) {
  return writeFile(filename, JSON.stringify(content, null, 2))
}
*/

function readGzipFile (filename, onMiss) {
  return readFile(filename, gzipOnMiss).then(buf => zlib.gunzip(buf).catch(() => {
    return clearFile(filename).then(() => readGzipFile(filename, onMiss))
  }))

  function gzipOnMiss () {
    return resolveCall(onMiss).then(result => zlib.gzip(result))
  }
}

function writeGzipFile (filename, content) {
  return writeFile(filename, zlib.gzip(content)).then(() => content)
}

function getUrlHash (toFetch) {
  return Bluebird.resolve(toFetch).then(toFetch => {
    const parsed = url.parse(toFetch)
{'fraction_non_alphanumeric': 0.13664259927797834, 'fraction_numerical': 0.0007220216606498195, 'mean_word_length': 5.211883408071749, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2586634', 'n_tokens_mistral': 1718, 'n_tokens_neox': 1702, 'n_words': 377}