<commit_before>var https = require('https');
var fs = require('fs');
var path = require('path');
var baseUrl = "https://nodejs.org/dist/";

var overwrite = false

exports.versions = function getVersions (options) {
  overwrite = options.overwrite

  https.get("https://nodejs.org/dist/index.json", function(res){
    if( res.statusCode !== 200){
      return console.log('response "' + res.status + '" from http://nodejs.org/dist/')
    }

    var responseData = ""

    res.on('data', function( data ){
      responseData = responseData + data
    })

    res.on('end', function(){
      JSON.parse(responseData).map(function(build){
        return build.version
      }).forEach(generateNodeFile)
    })
  })
}

// private

function generateNodeFile( version ){
  var shaUrl = baseUrl + version + "/";
  var shaData, shaLine, installLine, parts, filePath;

  if(/^(v4)/g.test(version)) {
    shaUrl += "SHASUMS256.txt";
    checksum = /[\da-zA-Z]{64}  node-v[\d]{1,2}\.[\d]{1,2}\.[\d]{1,2}.tar.gz/gi;
  } else {
    shaUrl += "SHASUMS.txt";
    checksum = /[\da-zA-Z]{40}  node-v[\d]{1,2}\.[\d]{1,2}\.[\d]{1,2}.tar.gz/gi;
  }

  https.get(shaUrl, function(res){
    if( res.statusCode !== 200){
      return console.log('response "' + res.status + '" from ' + shaUrl)
    }

    shaData = ''

    res.on('data', function(data){
      shaData = shaData + data
    });

    res.on('end', function(){
      shaLine = shaData.match(checksum)

      console.log(version, shaData, checksum)
      if(shaLine && shaLine.length){
        parts = shaLine[0].split('  ')
        //this is gnarly, oh well
        //install_package "node-v0.10.0" "http://nodejs.org/dist/v0.10.0/node-v0.10.0.tar.gz#7321266347dc1c47ed2186e7d61752795ce8a0ef"
        installLine = 'Xinstall_package "node-' + version + '" "' + baseUrl + version + '/' + parts[1] + '#'+parts[0]+'"\n'
        filePath = path.join(__dirname, '../share/node-build', version.substring(1))
        fs.exists(filePath,function(exists){

          if(!exists || overwrite){

            fs.writeFile(filePath, installLine, function(err){
              if(err)
                console.log(err)
              else
                console.log( version.substring(1) + ' file writen')
            })
          }
        })
      }
    })
  })
};<commit_msg>don't fetch sha info for files that already exist<commit_after>var https = require('https');
var fs = require('fs');
var path = require('path');
var baseUrl = "https://nodejs.org/dist/";

exports.versions = function getVersions (options) {
  https.get("https://nodejs.org/dist/index.json", function(res){
    if( res.statusCode !== 200){
      return console.log('response "' + res.status + '" from http://nodejs.org/dist/')
    }

    var responseData = ""

    res.on('data', function( data ){
      responseData = responseData + data
    })

    res.on('end', function(){
      JSON.parse(responseData)
      .map(function(build){
        var filename = filenameFor(build.version)
        return {
          version: build.version,
          file: {
            name: filename,
            exists: fs.existsSync(filename)
          }
        }
      })
      .filter(function(build){
        return !build.file.exists || options.overwrite
      })
      .forEach(generateNodeFile)
    })
  })
}

// private

function filenameFor(version){
  return path.join(__dirname, '../share/node-build', version.substring(1))
}

function generateNodeFile( build ){
  var version = build.version
  var shaUrl = baseUrl + version + "/"
  var shaData, shaLine, installLine, parts, filePath

  if(/^(v4)/g.test(version)) {
    shaUrl += "SHASUMS256.txt";
    checksum = /[\da-zA-Z]{64}  node-v[\d]{1,2}\.[\d]{1,2}\.[\d]{1,2}.tar.gz/gi;
  } else {
    shaUrl += "SHASUMS.txt";
    checksum = /[\da-zA-Z]{40}  node-v[\d]{1,2}\.[\d]{1,2}\.[\d]{1,2}.tar.gz/gi;
  }

  https.get(shaUrl, function(res){
    if( res.statusCode !== 200){
      return console.log('response "' + res.status + '" from ' + shaUrl)
    }

    shaData = ''

    res.on('data', function(data){
      shaData = shaData + data
    });

    res.on('end', function(){
      shaLine = shaData.match(checksum)

      if(shaLine && shaLine.length){
        parts = shaLine[0].split('  ')
        //this is gnarly, oh well
        //install_package "node-v0.10.0" "http://nodejs.org/dist/v0.10.0/node-v0.10.0.tar.gz#7321266347dc1c47ed2186e7d61752795ce8a0ef"
        installLine = 'Xinstall_package "node-' + version + '" "' + baseUrl + version + '/' + parts[1] + '#'+parts[0]+'"\n'
        filePath = path.join(__dirname, '../share/node-build', version.substring(1))
        fs.writeFile(filePath, installLine, function(err){
          if(err)
            console.log(err)
          else
            console.log( version.substring(1) + ' file writen')
        })
      }
    })
  })
};
{'fraction_non_alphanumeric': 0.1839056681836988, 'fraction_numerical': 0.029582126603227142, 'mean_word_length': 2.4934971098265897, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7621318', 'n_tokens_mistral': 1731, 'n_tokens_neox': 1642, 'n_words': 282}