<commit_before>/**
 * Created by tushar.mathur on 21/01/16.
 */

'use strict'
const Download = require('../index').Download
const Path = require('path')
const crypto = require('crypto')
const fs = require('fs')
const Rx = require('rx')

const getPath = x => Path.normalize(Path.join(__dirname, x))
exports.removeFile = (x) => Rx.Observable.fromCallback(fs.unlink)(getPath(x)).toPromise()
exports.createDownload = function (parameters) {
  var url = parameters.url
  var path = getPath(parameters.path)
  var mtd = new Download({path, url, strictSSL: false})
  var hash = crypto.createHash('sha1')
  return mtd.start().then(() => new Promise(resolve => fs.createReadStream(path)
    .on('data', x => hash.update(x))
    .on('end', () => resolve(hash.digest('hex')))
  ))
}
<commit_msg>refactor(test): move createFileDigest to a different function<commit_after>/**
 * Created by tushar.mathur on 21/01/16.
 */

'use strict'
const Download = require('../index').Download
const Path = require('path')
const crypto = require('crypto')
const fs = require('fs')
const Rx = require('rx')

const getPath = x => Path.normalize(Path.join(__dirname, x))
exports.removeFile = (x) => Rx.Observable.fromCallback(fs.unlink)(getPath(x)).toPromise()
const createFileDigest = exports.createFileDigest = path => {
  const hash = crypto.createHash('sha1')
  return new Promise(resolve => fs
    .createReadStream(path)
    .on('data', x => hash.update(x))
    .on('end', () => resolve(hash.digest('hex')))
  )
}
exports.createDownload = function (parameters) {
  var url = parameters.url
  var path = getPath(parameters.path)
  var mtd = new Download({path, url, strictSSL: false})
  return mtd.start().then(() => createFileDigest(path))
}

{'fraction_non_alphanumeric': 0.1733566026759744, 'fraction_numerical': 0.008144269924374637, 'mean_word_length': 4.771812080536913, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '860784', 'n_tokens_mistral': 565, 'n_tokens_neox': 550, 'n_words': 133}