<commit_before>var amqp = require('amqplib/callback_api');
const queueName = 'ad-location-queue';
import top500 from './top500';

function handleError(err) {
  if (err) {
    console.error(err);
    process.exit();
  }
}

amqp.connect('amqp://localhost', (err, conn) => {
  handleError(err);
  conn.createChannel((connErr, ch) => {
    handleError(connErr);
    ch.assertQueue(queueName, {
      durable: true
    });
    top500.forEach(site => {
      ch.sendToQueue(queueName, Buffer(site), {
        persistent: true
      });
      console.log(` [x] Sent ${site}`);
    });
    setTimeout(() => {
      conn.close();<commit_msg>Change to top 1M for pushSites<commit_after>var amqp = require('amqplib/callback_api');
let fs = require('fs');
const queueName = 'ad-location-queue';

let top1MRaw = fs.readFileSync('./data/top-1m.csv', 'utf-8');
let top1MLines = top1MRaw.split('\n');
let top1M = top1MLines.map(line => line.split(',')[1]);
top1M.splice(-1);

function handleError(err) {
  if (err) {
    console.error(err);
    process.exit();
  }
}

amqp.connect('amqp://localhost', (err, conn) => {
  handleError(err);
  conn.createChannel((connErr, ch) => {
    handleError(connErr);
    ch.assertQueue(queueName, {
      durable: true
    });
    top1M.forEach(site => {
      ch.sendToQueue(queueName, Buffer(site), {
        persistent: true
      });
      console.log(` [x] Sent ${site}`);
    });
    setTimeout(() => {
      conn.close();
{'fraction_non_alphanumeric': 0.18702553485162182, 'fraction_numerical': 0.014492753623188406, 'mean_word_length': 3.3154761904761907, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '607670', 'n_tokens_mistral': 514, 'n_tokens_neox': 513, 'n_words': 89}