<commit_before>module.change_code = 1; // this allows hotswapping of code (ignored in production)

var request = require("request");
var q = require('bluebird');
var browerifyCDN = "http://wzrd.in/bundle/"
var _ = require("underscore");

module.exports = function(obj, callback){
  var currentSnip = obj.currentSnip;
  var res = obj.res;
  var pkg = obj.pkg;
  var snip = pkg.latest.sniper;

  if(pkg.github === undefined || snip.srcs === undefined){
    res.status(500).send("github repo does not exist");  
    return;
  }

  // snip has extra config
  if(snip.extra != undefined && snip.extra[currentSnip] != undefined){
    // merge extra config
    for(var key in snip.extra[currentSnip]){
      snip[key] = _.uniq(_.union(snip[key], snip.extra[currentSnip][key]));
    }
  }

  // load browserified version
  if(!snip.noBrowserify){
    snip.js.push( browerifyCDN + pkg.name + "@" + pkg.version);
  }

  // expose other bundles
  if(snip.exposed !== undefined){
    for(var i=0;i<snip.exposed.length; i++){
      snip.js.push(browerifyCDN + snip.exposed[i] + "@latest");
    }
  }

  var baseLocal = convertGithubToRaw("/github/" + pkg.github.full_name + "/" + pkg.github.default_branch);

  // css
  // the user might specify alternative css locations
  if( snip.buildCSS !== undefined){
    snip.css = snip.buildCSS;
  }

  function translateURL(obj, base){
    // fix CDN urls
    if(obj.substring(0,2) === "//"){
      return "https:" + obj;
    }

    // translate root urls to github
    if(obj.charAt(0) === "/"){
      return base + obj;
    }
    return obj;
  }

  // translate all absolute URLs to github
  for(var i in snip.css){
    snip.css[i] = translateURL(snip.css[i], baseLocal);
  }

  // remove the build js (generated by browserify)
  var index = 0;
  while(index < snip.js.length){ 
    if(snip.js[index].substring(0,7) === "/build/"){
      snip.js.splice(index,1);
    } else {
      snip.js[index] = translateURL(snip.js[index], baseLocal);
      index++;
    }
  }

  // download snippets on-the-fly
  var ps = [];
  if(snip.srcs[currentSnip] === undefined){
    console.log("no js file found" , currentSnip, snip.srcs);  
    res.status(500).send("no js file found exist");  
    return;
  }

  // js
  snip.inlineScript = "";
  if(snip.srcs[currentSnip].js === undefined){
    console.log("no js file found" , currentSnip, snip.srcs[currentSnip]);  
  }else{
    var jsURL = convertGithubToRaw(snip.srcs[currentSnip].js.html_url);
    ps.push(new q.Promise(function(resolve,reject){
      request.get(jsURL, function (err, response, body) {

        snip.inlineScript = "";
        body = translateRelative(body,baseLocal,snip.snippets[0]);
        
        // inject yourDiv
        if(snip.hasNoHTML){
          snip.inlineScript = "var yourDiv = document.getElementById('yourDiv');\n";
        }

        snip.inlineScript += body;
        resolve();
      });
    }));
  }

  // html
  snip.inlineBody = "";
  if(snip.srcs[currentSnip].html !== undefined ){
    var htmlURL = convertGithubToRaw(snip.srcs[currentSnip].html.html_url);
    ps.push(new q.Promise(function(resolve,reject){
      request.get(htmlURL, function (err, response, body) {
        body = translateRelative(body,baseLocal,snip.snippets[0]);
        snip.inlineBody = body;
        resolve();
      });
    }));
  }else{
    snip.inlineBody = "<div id=yourDiv></div>";
    snip.hasNoHTML = true;
  }

  q.all(ps).then(function(){
    callback(snip);
  });
}

// TODO: apply dirty hacks on the snippets
// problem: access files from the snipper
function translateRelative(body, baseLocal, path){
  if(body.indexOf("./") >= 0){
    //var rawURL = "https://cors-anywhere.herokuapp.com/" + pkg.github.raw_url;
    var htmlUrl =  baseLocal + "/"+ path + "/";
    body = body.replace("../", baseLocal + "/");
    body = body.replace("./", htmlUrl);
  }
  return body;
}

function convertGithubToRaw(contentURL){
  contentURL = contentURL.replace("github.com", "raw.githubusercontent.com");
  contentURL = contentURL.replace("blob/", "");
  return contentURL;
}
<commit_msg>hack: get apinatomy working<commit_after>module.change_code = 1; // this allows hotswapping of code (ignored in production)

var request = require("request");
var q = require('bluebird');
var browerifyCDN = "http://wzrd.in/bundle/"
var _ = require("underscore");

module.exports = function(obj, callback){
  var currentSnip = obj.currentSnip;
  var res = obj.res;
  var pkg = obj.pkg;
  var snip = pkg.latest.sniper;
  var noBrowserify = !snip.noBrowserify || pkg.name != "apinatomy-core";

  if(pkg.github === undefined || snip.srcs === undefined){
    res.status(500).send("github repo does not exist");  
    return;
  }

  // snip has extra config
  if(snip.extra != undefined && snip.extra[currentSnip] != undefined){
    // merge extra config
    for(var key in snip.extra[currentSnip]){
      snip[key] = _.uniq(_.union(snip[key], snip.extra[currentSnip][key]));
    }
  }

  // load browserified version
  if(!noBrowserify){
    snip.js.push( browerifyCDN + pkg.name + "@" + pkg.version);
  }

  // expose other bundles
  if(snip.exposed !== undefined){
    for(var i=0;i<snip.exposed.length; i++){
      snip.js.push(browerifyCDN + snip.exposed[i] + "@latest");
    }
  }

  var baseLocal = convertGithubToRaw("/github/" + pkg.github.full_name + "/" + pkg.github.default_branch);

  // css
  // the user might specify alternative css locations
  if( snip.buildCSS !== undefined){
    snip.css = snip.buildCSS;
  }

  function translateURL(obj, base){
    // fix CDN urls
    if(obj.substring(0,2) === "//"){
      return "https:" + obj;
    }

    // translate root urls to github
    if(obj.charAt(0) === "/"){
      return base + obj;
    }
    return obj;
  }

  // translate all absolute URLs to github
  for(var i in snip.css){
    snip.css[i] = translateURL(snip.css[i], baseLocal);
  }

  // remove the build js (generated by browserify)
  var index = 0;
  while(index < snip.js.length){ 
    if(snip.js[index].substring(0,7) === "/build/"){
      snip.js.splice(index,1);
    } else {
      snip.js[index] = translateURL(snip.js[index], baseLocal);
      index++;
    }
  }

  // download snippets on-the-fly
  var ps = [];
  if(snip.srcs[currentSnip] === undefined){
    console.log("no js file found" , currentSnip, snip.srcs);  
    res.status(500).send("no js file found exist");  
    return;
  }

  // js
  snip.inlineScript = "";

  // TODO: dirty hack to get apinatomy working
  if(noBrowserify){
    snip.srcs[currentSnip].js = undefined;
  }

  if(snip.srcs[currentSnip].js === undefined){
    console.log("no js file found" , currentSnip, snip.srcs[currentSnip]);  
  }else{
    var jsURL = convertGithubToRaw(snip.srcs[currentSnip].js.html_url);
    ps.push(new q.Promise(function(resolve,reject){
      request.get(jsURL, function (err, response, body) {

        snip.inlineScript = "";
        body = translateRelative(body,baseLocal,snip.snippets[0]);
        
        // inject yourDiv
        if(snip.hasNoHTML){
          snip.inlineScript = "var yourDiv = document.getElementById('yourDiv');\n";
        }

        snip.inlineScript += body;
        resolve();
      });
    }));
  }

  // html
  snip.inlineBody = "";
  if(snip.srcs[currentSnip].html !== undefined ){
    var htmlURL = convertGithubToRaw(snip.srcs[currentSnip].html.html_url);
    ps.push(new q.Promise(function(resolve,reject){
      request.get(htmlURL, function (err, response, body) {
        body = translateRelative(body,baseLocal,snip.snippets[0]);
        snip.inlineBody = body;
        resolve();
      });
    }));
  }else{
    snip.inlineBody = "<div id=yourDiv></div>";
    snip.hasNoHTML = true;
  }

  q.all(ps).then(function(){
    callback(snip);
  });
}

// TODO: apply dirty hacks on the snippets
// problem: access files from the snipper
function translateRelative(body, baseLocal, path){
  if(body.indexOf("./") >= 0){
    //var rawURL = "https://cors-anywhere.herokuapp.com/" + pkg.github.raw_url;
    var htmlUrl =  baseLocal + "/"+ path + "/";
    body = body.replace("../", baseLocal + "/");
    body = body.replace("./", htmlUrl);
  }
  return body;
}

function convertGithubToRaw(contentURL){
  contentURL = contentURL.replace("github.com", "raw.githubusercontent.com");
  contentURL = contentURL.replace("blob/", "");
  return contentURL;
}

{'fraction_non_alphanumeric': 0.16419161676646707, 'fraction_numerical': 0.004311377245508982, 'mean_word_length': 3.012974531475252, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4476727', 'n_tokens_mistral': 2947, 'n_tokens_neox': 2854, 'n_words': 576}