<commit_before>var amqp = require('amqplib/callback_api');
let fs = require('fs');
import {init, getAdInfo, exit} from 'ad-info';

const queueName = 'ad-location-queue';

var amqp = require('amqplib/callback_api');

let perSiteInfo = {};
if (fs.existsSync('./perSiteInfo.json')) {
  perSiteInfo = JSON.parse(fs.readFileSync('./perSiteInfo.json', 'utf-8'));
}

console.log('[*] Waiting for messages. To exit press CTRL+C');

function handleError(err) {
  if (err) {
    console.error(err);
    process.exit();
  }
}

let sequence = init('./node_modules/ad-info/data/easylist.txt').then(() => {
  amqp.connect('amqp://localhost', (err, conn) => {
    handleError(err);
    conn.createChannel((connErr, ch) => {
      handleError(connErr);
      var q = 'ad-info-queue';
      ch.assertQueue(queueName, {durable: true});

      // Only send 1 message until previous one is ack'ed
      ch.prefetch(1);

      ch.consume(queueName, (msg) => {
        let siteHost = msg.content.toString();
        console.log(`http://${siteHost}`);

        getAdInfo(`http://${siteHost}`).then((info) => {
          perSiteInfo[siteHost] = info;
          fs.writeFileSync('./perSiteInfo.json', JSON.stringify(perSiteInfo));
          // Tell the queue we processed the site
          ch.ack(msg);
        }).catch((err) => {
          console.warn('Error obtaining info for site host:', siteHost, '. Error:', err);
          // Tell the queue we processed the site. This queue will be refilled periodically anyway.
          ch.ack(msg);
        });
      }, {
        noAck: false
      });
    });
  });
});<commit_msg>Only add ad info to JSON if there's ads on a page<commit_after>var amqp = require('amqplib/callback_api');
let fs = require('fs');
import {init, getAdInfo, exit} from 'ad-info';

const queueName = 'ad-location-queue';

var amqp = require('amqplib/callback_api');

let perSiteInfo = {};
if (fs.existsSync('./perSiteInfo.json')) {
  perSiteInfo = JSON.parse(fs.readFileSync('./perSiteInfo.json', 'utf-8'));
}

console.log('[*] Waiting for messages. To exit press CTRL+C');

function handleError(err) {
  if (err) {
    console.error(err);
    process.exit();
  }
}

let sequence = init('./node_modules/ad-info/data/easylist.txt').then(() => {
  amqp.connect('amqp://localhost', (err, conn) => {
    handleError(err);
    conn.createChannel((connErr, ch) => {
      handleError(connErr);
      var q = 'ad-info-queue';
      ch.assertQueue(queueName, {durable: true});

      // Only send 1 message until previous one is ack'ed
      ch.prefetch(1);

      ch.consume(queueName, (msg) => {
        let siteHost = msg.content.toString();
        console.log(`http://${siteHost}`);

        getAdInfo(`http://${siteHost}`).then((info) => {
          // To save space, only add information if there are ads on the page
          if (info.length > 0) {
            perSiteInfo[siteHost] = info;
            fs.writeFileSync('./perSiteInfo.json', JSON.stringify(perSiteInfo));
          }
          // Tell the queue we processed the site
          ch.ack(msg);
        }).catch((err) => {
          console.warn('Error obtaining info for site host:', siteHost, '. Error:', err);
          // Tell the queue we processed the site. This queue will be refilled periodically anyway.
          ch.ack(msg);
        });
      }, {
        noAck: false
      });
    });
  });
});
{'fraction_non_alphanumeric': 0.16978331849213416, 'fraction_numerical': 0.0020777678836449986, 'mean_word_length': 2.7320044296788484, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3699861', 'n_tokens_mistral': 1078, 'n_tokens_neox': 1065, 'n_words': 249}