<commit_before>var fs = require('fs');
var fsExtra = require('fs-extra');
var _ =  require('lodash');
var request = require('request');
var unzip = require('unzip');
var mv = require('mv');

var env = require('../env');

function getFileContent(tplFp, tplParams) {
  var tplContent = Utils.readFileSync(tplFp);

  if (!tplParams) {
    return tplContent;
  }
  return _.template(tplContent)(tplParams);
}

var Utils = {
  getTemplatePath: function(fp) {
    return `${__dirname}/../templates/${fp}.tpl`;
  },

  writeFileSync: function(targetFp, tplFp, tplParams) {
    var fileContent = getFileContent(tplFp, tplParams);
    fsExtra.outputFileSync(targetFp, fileContent);
  },

  readFileSync: function(fp, charset=null) {
    var content = fs.readFileSync(fp, charset);
    return content;
  },

  mkdirSync: function(dir) {
    try {
      fs.statSync(dir);
    } catch(err) {
      fs.mkdirSync(dir);
    }
  },

  downloadFile: function(fileUrl, targetFp, cb) {
    request(
      {
        url: fileUrl,
        encoding: null
      },
      function(err, resp, body) {
        if (err) {
          cb(err, null);
          return;
        }
        
        try {
          fs.writeFileSync(targetFp, body);
          cb(null, {fp: targetFp});
        } catch(err) {
          cb(err, null);
          return;
        }
      }
    );
  },

  unzipFile: function(srcFp, targetDir, cb) {
    var stream = fs.createReadStream(srcFp)
      .pipe(unzip.Extract({ path: targetDir }));

    stream.on('error', function (err) {
      cb(err, null);
      return;
    });

    stream.on('finish', function () {
      cb(null, true)
    });    
  },<commit_msg>Check download URL is valid or not<commit_after>var fs = require('fs');
var fsExtra = require('fs-extra');
var _ =  require('lodash');
var request = require('request');
var unzip = require('unzip');
var mv = require('mv');

var env = require('../env');

String.prototype.allReplace = function(obj) {
  var retStr = this;
  for (var x in obj) {
      retStr = retStr.replace(new RegExp(x, 'g'), obj[x]);
  }
  return retStr;
};

function getFileContent(tplFp, tplParams) {
  var tplContent = Utils.readFileSync(tplFp);

  if (!tplParams) {
    return tplContent;
  }
  return _.template(tplContent)(tplParams);
}

var Utils = {
  getTemplatePath: function(fp) {
    return `${__dirname}/../templates/${fp}.tpl`;
  },

  writeFileSync: function(targetFp, tplFp, tplParams) {
    var fileContent = getFileContent(tplFp, tplParams);
    fsExtra.outputFileSync(targetFp, fileContent);
  },

  readFileSync: function(fp, charset=null) {
    var content = fs.readFileSync(fp, charset);
    return content;
  },

  mkdirSync: function(dir) {
    try {
      fs.statSync(dir);
    } catch(err) {
      fs.mkdirSync(dir);
    }
  },

  downloadFile: function(fileUrl, targetFp, cb) {
    request(
      {
        url: fileUrl,
        encoding: null
      },
      function(err, resp, body) {
        if (err) {
          cb(err, null);
          return;
        }
        
        if (resp && resp.statusCode == 200) {
          try {
            fs.writeFileSync(targetFp, body);
            cb(null, {fp: targetFp});
          } catch(err) {
            cb(err, null);
            return;
          }
        } else {
          cb({
            name: 'Error',
            message: 'Invalid URL'
          }, null);
        }
      }
    );
  },

  unzipFile: function(srcFp, targetDir, cb) {
    var stream = fs.createReadStream(srcFp)
      .pipe(unzip.Extract({ path: targetDir }));

    stream.on('error', function (err) {
      cb(err, null);
      return;
    });

    stream.on('finish', function () {
      cb(null, true)
    });    
  },
{'fraction_non_alphanumeric': 0.1514741682445226, 'fraction_numerical': 0.0008114687584527996, 'mean_word_length': 2.3345356176735796, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3058056', 'n_tokens_mistral': 1262, 'n_tokens_neox': 1254, 'n_words': 255}