<commit_before>fs = require 'fs'
d3 = require 'd3'
async = require 'async'
request = require 'request'
path = require 'path'
shell = require 'shelljs'


skipExisting = true
#skipExisting = false
  
#base = __dirname + "/data/gists-clones/"
base = __dirname + "/data/gists-files/"

fs.mkdir base, -> # "data/gists", ->

param = process.argv[2]
if param
  if param.indexOf(".csv") > 0
    # list of ids to parse
    ids = d3.csv.parse(fs.readFileSync(singleId).toString())
  else
    singleId = param
    console.log "doing content for single block", singleId

gistMeta = JSON.parse fs.readFileSync('data/gist-meta.json').toString()
console.log gistMeta.length

timeouts = []

done = (err, pruned) ->
  console.log "done writing files"
  if timeouts.length
    console.log "timeouts", timeouts
  if singleId
    console.log "single id", singleId
    return
  if ids
    console.log "ids", ids
    return

gistCloner = (gist, gistCb) ->
  # I wanted to actually clone all the repositories but it seems to be less reliable.
  # we get rate limited if we use https:// git urls
  # and the ssh ones disconnect unexpectedly, probably some sort of rate limiting but it doesn't show in
  # curl https://api.github.com/rate_limit
  return gistCb() if ids && (gist.id not in ids)
  return gistCb() if singleId && (gist.id != singleId)
  token = require('./config.js').github.token
  console.log("token", token)
  folder = base + gist.id
  shell.cd(base)
  # TODO don't use token in clone url (git init; git pull with token)
  shell.exec 'git clone https://' + token + ' @gist.github.com/' + gist.id, (code, huh, message) ->
  #shell.exec 'git clone git@gist.github.com:' + gist.id, (code, huh, message) ->
    fs.lstat folder + "/.git", (err, stats) ->
      if err
        #console.log "err", err 
        # github timed out
        console.log "timeout", gist.id
        timeouts.push gist.id
        return gistCb()
      if stats.isDirectory()
        # we want to be able to pull recently modified gists
        return gistCb()
        ###
        console.log("exists, pulling", gist.id)
        shell.cd gist.id
        shell.exec 'git pull origin master', (code, huh, message) ->
          console.log("pulled", gist.id)
          return gistCb()
        ###
      else
        console.log "cloned", gist.id
        setTimeout ->
          gistCb()
        , 50 + Math.random() * 300


gistFetcher = (gist, gistCb) ->
  return gistCb() if ids && (gist.id not in ids)
  return gistCb() if singleId && (gist.id != singleId)
  #console.log "NOT RETURNING", gist.id, singleId
  fileNames = Object.keys gist.files
  folder = "data/gists/" + gist.id
  fs.mkdir folder, ->
  async.each fileNames, (fileName, fileCb) ->
    ext = path.extname(fileName).toLowerCase()
    filePath = folder + "/" + fileName
    if skipExisting
      try
        if fs.lstatSync(filePath)
          # if it exists we skip it
          console.log "skipping", gist.id, fileName
          return fileCb()
      catch e
        # otherwise we just continue
    if ext in [".html", ".js", ".coffee", ".md", ".json", ".csv", ".tsv", ".css"]
      file = gist.files[fileName]
      request.get file.raw_url, (err, response, body) -><commit_msg>make sure to use base dir name<commit_after>fs = require 'fs'
d3 = require 'd3'
async = require 'async'
request = require 'request'
path = require 'path'
shell = require 'shelljs'


skipExisting = true
#skipExisting = false
  
#base = __dirname + "/data/gists-clones/"
base = __dirname + "/data/gists-files/"

fs.mkdir base, -> # "data/gists", ->

param = process.argv[2]
if param
  if param.indexOf(".csv") > 0
    # list of ids to parse
    ids = d3.csv.parse(fs.readFileSync(singleId).toString())
  else
    singleId = param
    console.log "doing content for single block", singleId

gistMeta = JSON.parse fs.readFileSync('data/gist-meta.json').toString()
console.log gistMeta.length

timeouts = []

done = (err, pruned) ->
  console.log "done writing files"
  if timeouts.length
    console.log "timeouts", timeouts
  if singleId
    console.log "single id", singleId
    return
  if ids
    console.log "ids", ids
    return

gistCloner = (gist, gistCb) ->
  # I wanted to actually clone all the repositories but it seems to be less reliable.
  # we get rate limited if we use https:// git urls
  # and the ssh ones disconnect unexpectedly, probably some sort of rate limiting but it doesn't show in
  # curl https://api.github.com/rate_limit
  return gistCb() if ids && (gist.id not in ids)
  return gistCb() if singleId && (gist.id != singleId)
  token = require('./config.js').github.token
  console.log("token", token)
  folder = base + gist.id
  shell.cd(base)
  # TODO don't use token in clone url (git init; git pull with token)
  shell.exec 'git clone https://' + token + ' @gist.github.com/' + gist.id, (code, huh, message) ->
  #shell.exec 'git clone git@gist.github.com:' + gist.id, (code, huh, message) ->
    fs.lstat folder + "/.git", (err, stats) ->
      if err
        #console.log "err", err 
        # github timed out
        console.log "timeout", gist.id
        timeouts.push gist.id
        return gistCb()
      if stats.isDirectory()
        # we want to be able to pull recently modified gists
        return gistCb()
        ###
        console.log("exists, pulling", gist.id)
        shell.cd gist.id
        shell.exec 'git pull origin master', (code, huh, message) ->
          console.log("pulled", gist.id)
          return gistCb()
        ###
      else
        console.log "cloned", gist.id
        setTimeout ->
          gistCb()
        , 50 + Math.random() * 300


gistFetcher = (gist, gistCb) ->
  return gistCb() if ids && (gist.id not in ids)
  return gistCb() if singleId && (gist.id != singleId)
  #console.log "NOT RETURNING", gist.id, singleId
  fileNames = Object.keys gist.files
  folder = base + gist.id
  fs.mkdir folder, ->
  async.each fileNames, (fileName, fileCb) ->
    ext = path.extname(fileName).toLowerCase()
    filePath = folder + "/" + fileName
    if skipExisting
      try
        if fs.lstatSync(filePath)
          # if it exists we skip it
          console.log "skipping", gist.id, fileName
          return fileCb()
      catch e
        # otherwise we just continue
    if ext in [".html", ".js", ".coffee", ".md", ".json", ".csv", ".tsv", ".css"]
      file = gist.files[fileName]
      request.get file.raw_url, (err, response, body) ->
{'fraction_non_alphanumeric': 0.13329180940516974, 'fraction_numerical': 0.003114294612270321, 'mean_word_length': 2.6390934844192633, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 31, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1313232', 'n_tokens_mistral': 2154, 'n_tokens_neox': 2117, 'n_words': 658}