<commit_before>var _ = require('lodash');
var fs = require('fs');
var geoip = require('geoip-lite');
var crawler = require('../src/crawler.js');
var normalizeIpp = crawler.normalizeIpp;
var normalizePubKey = crawler.normalizePubKey;

var rawCrawl = require('../' + process.argv.slice(2)[0]);

function graphify(rawCrawl) {

  var out = {nodes: [], links: [], iplinks: []}

  // storage
  var nodes = []
  var links = {}
  var indices = {}
  var keyToIp = {}

  // Collect all unique nodes and save their indices
  for (var ipp in rawCrawl.data) {
    indices[ipp] = nodes.length
    if (nodes.length == 0) {
      nodes.push({ipp: ipp, connections: 0, entry: 1})
    } else {
      nodes.push({ipp: ipp, connections: 0, entry: 0})
    }
    keyToIp[ipp]
  }

  // For each unique node
  for(i in nodes) {
    node = nodes[i];
    ipp = node.ipp;
    peers = rawCrawl.data[ipp].overlay.active;

    // For each (not necessarily unique) peer
    for (p_index in peers) {
      peer = peers[p_index];

      // Get ipp
      try {
        peer_ipp = normalizeIpp(peer.ip, peer.port);
      } catch (err) {
        peer_ipp = undefined;
      }
      peer_i = indices[peer_ipp];

      // If it has an ipp
      if (peer_ipp && peer_i !== undefined) {
        peer_pk = normalizePubKey(peer.public_key);
        peer_v = peer.version;
        peer_t = peer.type;

        // Fill out info in unique node list
        peer_node = nodes[peer_i];

        //console.error(peer_ipp)
        if (!peer_node.pk) {
          peer_node.pk = peer_pk
        }
        if (!peer_node.v) {
          peer_node.v = peer_v
        }
        if (!peer_node.loc) {
          geoloc = geoip.lookup(peer_ipp.split(':')[0])
          peer_node.loc = geoloc.country + '_' + geoloc.city
        }

        // Make link
        var link = undefined
        if (peer_t) {
          // Get link
          if (peer_t == "in") {
            link = [i, peer_i]
          }
          else if (peer_t == "out") {
            link = [peer_i, i]
          }
          else if (peer_t == "peer") {
            if (peer.ip) {
              if (peer.ip.split(":").length == 2)
                link = [peer_i, i]
              else
                link = [i, peer_i]
            }
          }

          if (peer_t && !links[peer_t]) {
            links[link] = 1
          }
        }
<commit_msg>remove unused function keyToIp and minor fixes<commit_after>var _ = require('lodash');
var fs = require('fs');
var geoip = require('geoip-lite');
var crawler = require('../src/crawler.js');
var normalizeIpp = crawler.normalizeIpp;
var normalizePubKey = crawler.normalizePubKey;

var rawCrawl = require('../' + process.argv.slice(2)[0]);

function graphify(rawCrawl) {

  var out = {nodes: [], links: [], iplinks: []}

  // storage
  var nodes = []
  var links = {}
  var indices = {}

  // Collect all unique nodes and save their indices
  for (var ipp in rawCrawl.data) {
    indices[ipp] = nodes.length
    if (nodes.length == 0) {
      nodes.push({ipp: ipp, connections: 0, entry: 1})
    } else {
      nodes.push({ipp: ipp, connections: 0, entry: 0})
    }
  }

  // For each unique node
  for(i in nodes) {
    node = nodes[i];
    ipp = node.ipp;
    peers = rawCrawl.data[ipp].overlay.active;

    // For each (not necessarily unique) peer
    for (p_index in peers) {
      peer = peers[p_index];

      // Get ipp
      try {
        peer_ipp = normalizeIpp(peer.ip, peer.port);
      } catch (err) {
        peer_ipp = undefined;
      }
      peer_i = indices[peer_ipp];

      // If it has an ipp
      if (peer_ipp && peer_i !== undefined) {
        peer_pk = normalizePubKey(peer.public_key);
        peer_v = peer.version;
        peer_t = peer.type;

        // Fill out info in unique node list
        peer_node = nodes[peer_i];

        //console.error(peer_ipp)
        if (!peer_node.pk) {
          peer_node.pk = peer_pk
        }
        if (!peer_node.v) {
          peer_node.v = peer_v
        }
        if (!peer_node.loc) {
          geoloc = geoip.lookup(peer_ipp.split(':')[0])
          peer_node.loc = geoloc.country + '_' + geoloc.city
        }

        // Make link
        var link = undefined
        if (peer_t) {
          // Get link
          if (peer_t == "in") {
            link = [i, peer_i]
          }
          else if (peer_t == "out") {
            link = [peer_i, i]
          }
          else if (peer_t == "peer") {
            if (peer.ip) {
              if (peer.ip.split(":").length == 2)
                link = [peer_i, i]
              else
                link = [i, peer_i]
            }
          }

          links[link] = 1
        }

{'fraction_non_alphanumeric': 0.13902596009440035, 'fraction_numerical': 0.004290924694271616, 'mean_word_length': 1.7136204889406286, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2683482', 'n_tokens_mistral': 1607, 'n_tokens_neox': 1541, 'n_words': 378}