<commit_before>path = require "path"
superagent = require "superagent"
comerr = require "comerr"
async = require "async"
detective = require "detective"
paperclip = require "paperclip"
parser = require "paperclip/lib/parser"

crypto = require('crypto')

exports.compile = (files, useCompiler, complete) ->

  if typeof useCompiler is "function"
    complete = useCompiler
    useCompiler = true

  addLocalDeps files

  addRemoteFiles files, (err) ->
    if err
      return complete err
    try 
      script = parse files
      if useCompiler
        fn = compile script
        
      complete null, fn, script
    catch e
      complete e


getFilesHash = (files) -> crypto.createHash('md5').update(JSON.stringify(files)).digest('hex')


addLocalDeps = (files) ->
  files.forEach (file) ->
    file.deps = detective file.content


scanRemoteDeps = (files) ->
  remoteDeps = []

cdn = "http://browserify-cdn.herokuapp.com"
#cdn = "http://wzrd.in"

_remoteModuleCache = {}

addRemoteFiles = (files, complete) ->
  remoteDeps = {}

  for file in files
    for dep in file.deps
      unless ~dep.indexOf("/")
        depParts = dep.split("@")
        name = depParts.shift()
        version = depParts.shift() || "latest"
        moduleName = name + (if version is "latest" then "" else "@" + version)
<commit_msg>use wzrd<commit_after>path = require "path"
superagent = require "superagent"
comerr = require "comerr"
async = require "async"
detective = require "detective"
paperclip = require "paperclip"
parser = require "paperclip/lib/parser"

crypto = require('crypto')

exports.compile = (files, useCompiler, complete) ->

  if typeof useCompiler is "function"
    complete = useCompiler
    useCompiler = true

  addLocalDeps files

  addRemoteFiles files, (err) ->
    if err
      return complete err
    try 
      script = parse files
      if useCompiler
        fn = compile script
        
      complete null, fn, script
    catch e
      complete e


getFilesHash = (files) -> crypto.createHash('md5').update(JSON.stringify(files)).digest('hex')


addLocalDeps = (files) ->
  files.forEach (file) ->
    file.deps = detective file.content


scanRemoteDeps = (files) ->
  remoteDeps = []

cdn = "http://browserify-cdn.herokuapp.com"
cdn = "http://wzrd.in"

_remoteModuleCache = {}

addRemoteFiles = (files, complete) ->
  remoteDeps = {}

  for file in files
    for dep in file.deps
      unless ~dep.indexOf("/")
        depParts = dep.split("@")
        name = depParts.shift()
        version = depParts.shift() || "latest"
        moduleName = name + (if version is "latest" then "" else "@" + version)

{'fraction_non_alphanumeric': 0.12318840579710146, 'fraction_numerical': 0.0007627765064836003, 'mean_word_length': 2.9984756097560976, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6785095', 'n_tokens_mistral': 819, 'n_tokens_neox': 794, 'n_words': 223}