<commit_before>var fs = require('fs');
var JSONStream = require('JSONStream');
var es = require('event-stream');
var createGraph = require('ngraph.graph');

module.exports = loadGraph;

function loadGraph(fileName, done) {
  if (!fs.existsSync(fileName)) {
    console.error('No data file: ' + fileName);
    console.error('Make sure to index youtube first with `node index.js` command');
    process.exit(-1);
  }
  var graph = createGraph({uniqueLinkId: false});
  var parser = JSONStream.parse();
  var ids = new Set();
  fs.createReadStream(fileName)
    .pipe(parser)
    .pipe(es.mapSync(addChannel))
    .on('end', allDone);

  function addChannel(pkg) {
    if (ids.has(pkg.id)) {
      console.log('Dupe: ' + pkg.id);
    }
    ids.add(pkg.id);
    if (!pkg.title) return;
    if (pkg.title[0] === '#') return;

    graph.addNode(pkg.id);
    var related = pkg.related;
    if (!related || related.length === 0) return;
    if (pkg.relatedTitle === 'Popular channels' ||
       pkg.relatedTitle === 'Related channels') {
//      console.log('no links for ' + pkg.id);
      return;
    }
    for (var i = 0; i < related.length; ++i) {
      graph.addLink(pkg.id, related[i]);
    }
  }<commit_msg>Added reasons behind ignoring channels<commit_after>var fs = require('fs');
var JSONStream = require('JSONStream');
var es = require('event-stream');
var createGraph = require('ngraph.graph');

module.exports = loadGraph;

function loadGraph(fileName, done) {
  if (!fs.existsSync(fileName)) {
    console.error('No data file: ' + fileName);
    console.error('Make sure to index youtube first with `node index.js` command');
    process.exit(-1);
  }
  var graph = createGraph({uniqueLinkId: false});
  var parser = JSONStream.parse();
  var ids = new Set();
  fs.createReadStream(fileName)
    .pipe(parser)
    .pipe(es.mapSync(addChannel))
    .on('end', allDone);

  function addChannel(pkg) {
    if (ids.has(pkg.id)) {
      console.log('Dupe: ' + pkg.id);
    }
    ids.add(pkg.id);
    if (!pkg.title) return;
    // Channels with # as a first letter are auto-generated channels. Ignore them
    if (pkg.title[0] === '#') return;

    graph.addNode(pkg.id);
    var related = pkg.related;
    if (!related || related.length === 0) return;
    // Popular and Related channels are suggested by YouTube when user didn't include any.
    // We will ignore them
    if (pkg.relatedTitle === 'Popular channels' ||
       pkg.relatedTitle === 'Related channels') {
      return;
    }
    for (var i = 0; i < related.length; ++i) {
      graph.addLink(pkg.id, related[i]);
    }
  }
{'fraction_non_alphanumeric': 0.16254826254826255, 'fraction_numerical': 0.003088803088803089, 'mean_word_length': 3.2198697068403908, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '867427', 'n_tokens_mistral': 848, 'n_tokens_neox': 832, 'n_words': 214}