<commit_before>var outFileName = 'youtube-user.json';
var fs = require('fs');
var JSONStream = require('JSONStream');
var es = require('event-stream');
var createQueue = require('./lib/queue.js');
var startCrawl = require('./lib/startCrawl.js');
var outgoing;
var MAX_RUNNING = 5;
// we will use bloom filter to check whether website is already indexed.
var bloom = initBloomFilter();
readProcessedFile(outFileName, crawl);


function crawl(ids, processedRows) {
  var queue = createQueue(ids, MAX_RUNNING);
  startCrawl(queue, onProcessed, processedRows);

  function onProcessed(page) {
    bloom.add(page.id);
    write(page);
    enqueueRelated(page.related, queue);
  }
}
function write(page) {
  if(!outgoing) {
    createOutStream();
  }

  outgoing.write(page);
}

function createOutStream() {
  outgoing = JSONStream.stringify(false);
  var fileStream = fs.createWriteStream(outFileName, {
    encoding: 'utf8',
    flags: 'a'
  });
  outgoing.pipe(fileStream);
}

function enqueueRelated(related, queue) {
  if (!related || related.length === 0) return;
  for (var i = 0; i < related.length; ++i) {
    if (!bloom.test(related[i]) && queue.length < 100000) {
      // we will cap our queue at 100,000 channels. Subsequent reruns should catch missing channels
      queue.push(related[i]);
      // make sure we are not adding anything already queued.
      bloom.add(related[i]);
    }
  }
}

function readProcessedFile(fileName, done) {
  if (!fs.existsSync(fileName)) {
    startFromTotalBiscuit(done);
    return;
  }
  var queue = [];
  var processedRows = 0;
<commit_msg>Should be a function<commit_after>var outFileName = 'youtube-user.json';
var fs = require('fs');
var JSONStream = require('JSONStream');
var es = require('event-stream');
var createQueue = require('./lib/queue.js');
var startCrawl = require('./lib/startCrawl.js');
var outgoing;
var MAX_RUNNING = 5;
// we will use bloom filter to check whether website is already indexed.
var bloom = initBloomFilter();
readProcessedFile(outFileName, crawl);


function crawl(ids, processedRows) {
  var queue = createQueue(ids, MAX_RUNNING);
  startCrawl(queue, onProcessed, processedRows);

  function onProcessed(page) {
    bloom.add(page.id);
    write(page);
    enqueueRelated(page.related, queue);
  }
}
function write(page) {
  if(!outgoing) {
    createOutStream();
  }

  outgoing.write(page);
}

function createOutStream() {
  outgoing = JSONStream.stringify(false);
  var fileStream = fs.createWriteStream(outFileName, {
    encoding: 'utf8',
    flags: 'a'
  });
  outgoing.pipe(fileStream);
}

function enqueueRelated(related, queue) {
  if (!related || related.length === 0) return;
  for (var i = 0; i < related.length; ++i) {
    if (!bloom.test(related[i]) && queue.getLength() < 100000) {
      // we will cap our queue at 100,000 channels. Subsequent reruns should catch missing channels
      queue.push(related[i]);
      // make sure we are not adding anything already queued.
      bloom.add(related[i]);
    }
  }
}

function readProcessedFile(fileName, done) {
  if (!fs.existsSync(fileName)) {
    startFromTotalBiscuit(done);
    return;
  }
  var queue = [];
  var processedRows = 0;

{'fraction_non_alphanumeric': 0.13819095477386933, 'fraction_numerical': 0.010678391959798994, 'mean_word_length': 4.0395569620253164, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4948205', 'n_tokens_mistral': 1066, 'n_tokens_neox': 1011, 'n_words': 261}