<commit_before>var fs = require('fs');
var _ = require('underscore');
var spawn = require('child_process').spawn;
var prc = spawn('npm', ['ls', '--json', '--long']);
var data = '';
var includes = ["name", "version", "licenses", "license", "dependencies"];
var modules;
var request = require('request');
var path = require('path');
var appDir = path.dirname(require.main.filename);

var host = 'http://127.0.0.1:8008/';
var uri = 'http://127.0.0.1:8008/post'

var parseModule = function(data) {

  var modules = data.dependencies;
  // this should come from a remote source .. 
  var rules = {
    "MIT": true,
    "GPL": false
  };

  var Node = function(value, parent) {
    this.value = value;
    this.parent = parent;
  };

  // If data.licenses and a license fails the test, mark warning
  var markNodeSafety = function(data) {
    if (data.licenses) {
      var passes = true;
      for (var i = 0; i < data.licenses.length; i++) {
        if (rules[data.licenses[i]] === false) {
          passes = false;
          break;
        } else if (!rules[data.licenses[i]] || rules[data.licenses[i]] !== true) {
          passes = "warn";
          break;
        }
      }
      data.passes = passes;
    } else if (data.license) {
      if (rules[data.license] === true) {
        data.passes = true;
      } else if (!rules[data.license] || rules[data.license] !== true) {
        data.passes = "warn";
      } else {
        data.passes = false;
      }
    }
  };

  // Mark parent node safety status
  var markParentNode = function(parent, data) {
    var currentParent = parent;
    if (data.passes !== true) {
      while (currentParent) {
        if (data.passes === false) {
          parent.passes = false;
        } else if (data.passes === "warn" && parent.value && parent.passes !== false) {
          parent.passes = "warn";
        }
        currentParent = parent.parent;
      }
    } else if (!parent.passes) {
      parent.passes = true;
    }
  };

  var makeNode = function(name, data, parent) {
    // Modify data object
    data.name = name;
    var dependencies = data.dependencies;
    data.dependencies = [];

    // Create child node for each dependency that points to data object as parent
    Object.keys(dependencies).forEach(function(depName) {
      makeNode(depName, dependencies[depName], data);
    });

    // Determine data object safety
    markNodeSafety(data);

    if (parent) {
      if (data.passes !== true) {
        markParentNode(parent, data);
      }
      parent.dependencies.push(data);
    }
  }

  // Begin parsing data
  Object.keys(modules).forEach(function(modKey) {
    makeNode(modKey, modules[modKey], null);
    // console.log(util.inspect(modules, {depth: null}));
  });

  return modules;

};

module.exports = function() {

  prc.stdout.setEncoding('utf8');
  prc.stdout.on('data', function(chunk) {
    data += chunk;
  });
<commit_msg>Update data structure, handle unknown licenses<commit_after>var fs = require('fs');
var _ = require('underscore');
var spawn = require('child_process').spawn;
var prc = spawn('npm', ['ls', '--json', '--long']);
var data = '';
var includes = ["name", "version", "licenses", "license", "dependencies"];
var modules;
var request = require('request');
var path = require('path');
var appDir = path.dirname(require.main.filename);

var host = 'http://127.0.0.1:8008/';
var uri = 'http://127.0.0.1:8008/post'

var parseModule = function(data) {

  var modules = data.dependencies;
  // this should come from a remote source .. 
  var rules = {
    "MIT": true,
    "GPL": false
  };

  // If data.licenses and a license fails the test, mark warning
  var markNodeSafety = function(data) {
    if (!data.license && !data.licenses) {
      data.license = "unknown"
    }
    if (data.licenses) {
      var passes = true;
      for (var i = 0; i < data.licenses.length; i++) {
        if (rules[data.licenses[i].type] === false) {
          passes = false;
          break;
        } else if (!rules[data.licenses[i].type] || rules[data.licenses[i].type] !== true) {
          passes = "warn";
          break;
        }
      }
      data.passes = passes;
    } else if (data.license) {
      if (rules[data.license] === true) {
        data.passes = true;
      } else if (!rules[data.license] || rules[data.license] !== true) {
        data.passes = "warn";
      } else {
        data.passes = false;
      }
    }
  };

  // Mark parent node safety status
  var markParentNode = function(parent, data) {
    if (data.passes === false) {
      parent.passes = false;
    } else if (data.passes === "warn" && parent.passes !== false) {
      parent.passes = "warn";
    }
    if (parent.parent) {
      markParentNode(parent.parent, data);
    }
  };

  var makeNode = function(name, data, parent) {
    // Modify data object
    data.name = name;
    var dependencies = data.dependencies;
    data.dependencies = [];
    data.parent = parent

    // Determine data object safety
    markNodeSafety(data);
      // Create child node for each dependency that points to data object as parent
    Object.keys(dependencies).forEach(function(depName) {
      makeNode(depName, dependencies[depName], data);
    });

    if (parent){
      if (data.passes !== true){
        markParentNode(parent, data);
      }
      parent.dependencies.push(data);
    }
  }

  // Begin parsing data
  Object.keys(modules).forEach(function(modKey) {
    makeNode(modKey, modules[modKey], null);
  });

  return modules;

};

module.exports = function() {

  prc.stdout.setEncoding('utf8');
  prc.stdout.on('data', function(chunk) {
    data += chunk;
  });

{'fraction_non_alphanumeric': 0.1581750399431919, 'fraction_numerical': 0.0078111130836144155, 'mean_word_length': 2.5931122448979593, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1703785', 'n_tokens_mistral': 1813, 'n_tokens_neox': 1752, 'n_words': 443}