<commit_before>'use strict';


/** Proxy and cache a file and related headers
 *
 *  @copyright  Copyright (C) 2015 by Yieme
 *  @module     proxyCacheFile
 */

'use strict';
var _              = require('lodash')
var fs             = require('fs')
var mkdirp         = require('mkdirp')
var MD5            = require('MD5')
var request        = require('request')
var path           = require('path')
var forwardHeaders = ['content-type', 'content-encoding']
var options        = {
  dir:  '/tmp' // falsely to disable file cache
}

var ProxyCacheFileError = require('make-error')('ProxyCacheFileError')


/*
@param {object} request || config
- request contains: { url: 'https://domain.com/path/to/file', gzip: false }
- config contails:  { cacheDir: '/tmp' }
@param {function} callback
@return {object} result - { header: [{name:'', value: ''}], data: data || , piped: pipeData }
*/
function proxyCacheFile(req, callback) {
  if ('string' === typeof req) req = { url: req }
  req = req || {}
  if ('undefined' !== typeof req.dir) {
    if (options.dir != req.dir) {
      options.dir = req.dir
      if (options.dir) {
        mkdirp.sync(options.dir)
      }
      options = _.extend(options, req)
    }
    if (!req.url) {
      options = _.extend(options, req)
      return proxyCacheFile
    }
  }
  req.dir = req.dir || options.dir

  function cacheFile(filePath, headers, data) {
    var writeStream = fs.createWriteStream(filePath)
    writeStream.write(data)
    writeStream.end()
    fs.writeFile(filePath + '.json', JSON.stringify(headers), 'utf8', function(err) {
      if (err) callback(new ProxyCacheFileError(err))
    })
  }

  function tryProxy(filePath) {
    request({ method: 'GET', gzip: req.gzip, uri: req.url })
    .on('error', function(err) {
      callback(err)
    })
    .on('response', function(response) {
      var headers       = response.headers
      var fileHeaders    = []
      for (var i=0, len = forwardHeaders.length; i < len; i++) {
        var headerName  = forwardHeaders[i]
        var header      = headers[headerName]
        if (header) {
          fileHeaders.push({ name: headerName, value: header })
        }
      }
//      if (cache.control) fileHeaders.push({ name: 'cache-control', value: 'public, max-age=' + cache.control })
      response.on('data', function(data) {
//        console.log(req.url, 'received ' + data.length + ' bytes of compressed data', typeof data) // TODO: add logging option
        if (filePath) cacheFile(filePath, fileHeaders, data)
        callback(null, { headers: fileHeaders, data: data, isBuffer: true })
      })
    })
  }

  function tryFileCache(filePath) {
    fs.exists(filePath, function(exists) {
      if (exists) {
        fs.readFile(filePath + '.json', 'utf8', function(err, headers) {
          if (err) throw new Error(err)
//          console.log(filePath, 'from cache') // TODO: add logging option
          headers = JSON.parse(headers)
          var readStream = fs.createReadStream(filePath)
          callback(null, { headers: headers, data: readStream })
        })
      } else {
        tryProxy(filePath)
      }
    })
  }

  if (!req.url) {
    callback(new ProxyCacheFileError('Missing URL'))
  } else if (req.url.indexOf('..') < 0) {
		if (req.dir) {
      tryFileCache(path.normalize(req.dir + '/' + MD5(req.url + (req.gzip) ? '.gz' : '')))
		} else {
      tryProxy(req)
    }
	} else {
		callback(new ProxyCacheFileError('Invalid URL: ' + req.url))
	}
}


module.exports = proxyCacheFile<commit_msg>optionally return requested url with response<commit_after>'use strict';


/** Proxy and cache a file and related headers
 *
 *  @copyright  Copyright (C) 2015 by Yieme
 *  @module     proxyCacheFile
 */

'use strict';
var _              = require('lodash')
var fs             = require('fs')
var mkdirp         = require('mkdirp')
var MD5            = require('MD5')
var request        = require('request')
var path           = require('path')
var forwardHeaders = ['content-type', 'content-encoding']
var options        = {
  dir:       '/tmp', // falsely to disable file cache
}

var ProxyCacheFileError = require('make-error')('ProxyCacheFileError')


/*
@param {object} request || config
- request contains: { url: 'https://domain.com/path/to/file', gzip: false, returnUrl: false }
- config contails:  { cacheDir: '/tmp' }
@param {function} callback
@return {object} result - { header: [{name:'', value: ''}], data: data || , piped: pipeData }
*/
function proxyCacheFile(req, callback) {
  if ('string' === typeof req) req = { url: req }
  req = req || {}
  if ('undefined' !== typeof req.dir) {
    if (options.dir != req.dir) {
      options.dir = req.dir
      if (options.dir) {
        mkdirp.sync(options.dir)
      }
      options = _.extend(options, req)
    }
    if (!req.url) {
      options = _.extend(options, req)
      return proxyCacheFile
    }
  }
  req.dir = req.dir || options.dir

  function cacheFile(filePath, headers, data) {
    var writeStream = fs.createWriteStream(filePath)
    writeStream.write(data)
    writeStream.end()
    fs.writeFile(filePath + '.json', JSON.stringify(headers), 'utf8', function(err) {
      if (err) callback(new ProxyCacheFileError(err))
    })
  }

  function tryProxy(filePath) {
    request({ method: 'GET', gzip: req.gzip, uri: req.url })
    .on('error', function(err) {
      callback(err)
    })
    .on('response', function(response) {
      var headers       = response.headers
      var fileHeaders    = []
      for (var i=0, len = forwardHeaders.length; i < len; i++) {
        var headerName  = forwardHeaders[i]
        var header      = headers[headerName]
        if (header) {
          fileHeaders.push({ name: headerName, value: header })
        }
      }
//      if (cache.control) fileHeaders.push({ name: 'cache-control', value: 'public, max-age=' + cache.control })
      response.on('data', function(data) {
//        console.log(req.url, 'received ' + data.length + ' bytes of compressed data', typeof data) // TODO: add logging option
        if (filePath) cacheFile(filePath, fileHeaders, data)
        var result = { headers: fileHeaders, data: data, isBuffer: true }
        if (req.returnUrl) result.url = req.url
        callback(null, result)
      })
    })
  }

  function tryFileCache(filePath) {
    fs.exists(filePath, function(exists) {
      if (exists) {
        fs.readFile(filePath + '.json', 'utf8', function(err, headers) {
          if (err) throw new Error(err)
//          console.log(filePath, 'from cache') // TODO: add logging option
          headers = JSON.parse(headers)
          var readStream = fs.createReadStream(filePath)
          var result = { headers: headers, data: readStream }
          if (req.returnUrl) result.url = req.url
          callback(null, result)
        })
      } else {
        tryProxy(filePath)
      }
    })
  }

  if (!req.url) {
    callback('Missing req.url')
  } else if (req.url.indexOf('..') < 0) {
		if (req.dir) {
      tryFileCache(path.normalize(req.dir + '/' + MD5(req.url + (req.gzip) ? '.gz' : '')))
		} else {
      tryProxy(req)
    }
	} else {
		callback('Invalid req.url: ' + req.url)
	}
}


module.exports = proxyCacheFile
{'fraction_non_alphanumeric': 0.1506107717934481, 'fraction_numerical': 0.003053858967240422, 'mean_word_length': 2.5827946295375437, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3149652', 'n_tokens_mistral': 2288, 'n_tokens_neox': 2247, 'n_words': 558}