<commit_before>var fs = require('fs');
var url = require('url');

var async = require('async');
var request = require('request');
var argv = require('optimist').argv;

var MailListener = require('mailListener');
var MailWorker = require('mailWorker');
var MemoryQueue = require('memoryQueue');
var UIDStore = require('UIDStore');
var config = require('config');

function start(user, pass, lastUID, rulesPath) {
  // start listening for new messages
  var mailListener = new MailListener({
    user: user,
    password: <PASSWORD>,
    host: 'imap.gmail.com',
    port: 993,
    tls: true,
    tlsOptions: { rejectUnauthorized: false }
  }, new UIDStore('mem', lastUID));

  var Q = new MemoryQueue(user);
  mailListener.on('message', function(msg) {
    Q.enqueue(msg, function(err) {
      if (err) console.error('enqueue error', err);
    });
  });
  mailListener.start();

  // start a worker to handle the messages
  var mailWorker = new MailWorker(Q, rulesPath);
  mailWorker.start();
}

if (!module.parent) {
  var etcdHost = argv['etcd-host'];
  config.load(etcdHost? 'etcd':null, etcdHost, function(err) {
    if (err) {
      console.error('error loading config', err);
      process.exit(2);
    }
    start(config.IMAP_USER, config.IMAP_PASS,
      config.IMAP_UID, config.IMAP_RULES);
  });
}
<commit_msg>move to etcdUIDStore<commit_after>var fs = require('fs');
var url = require('url');

var async = require('async');
var request = require('request');
var argv = require('optimist').argv;

var MailListener = require('mailListener');
var MailWorker = require('mailWorker');
var MemoryQueue = require('memoryQueue');
var UIDStore = require('UIDStore');
var config = require('config');

function start(user, pass, uidConfig, rulesPath) {
  // start listening for new messages
  var mailListener = new MailListener({
    user: user,
    password: pass,
    host: 'imap.gmail.com',
    port: 993,
    tls: true,
    tlsOptions: { rejectUnauthorized: false }
  }, new UIDStore('etcd', uidConfig));

  var Q = new MemoryQueue(user);
  mailListener.on('message', function(msg) {
    Q.enqueue(msg, function(err) {
      if (err) console.error('enqueue error', err);
    });
  });
  mailListener.start();

  // start a worker to handle the messages
  var mailWorker = new MailWorker(Q, rulesPath);
  mailWorker.start();
}

if (!module.parent) {
  var etcdHost = argv['etcd-host'];
  config.load(etcdHost? 'etcd':null, etcdHost, function(err) {
    if (err) {
      console.error('error loading config', err);
      process.exit(2);
    }
    var user = config.IMAP_USER;
    var pass = config.IMAP_PASS;
    var uidConfig = etcdHost? {host: etcdHost} : config.IMAP_UID;
    var rules = config.IMAP_RULES;
    start(user, pass, uidConfig, rules);
  });
}

{'fraction_non_alphanumeric': 0.15248098515030786, 'fraction_numerical': 0.002897500905469033, 'mean_word_length': 4.012704174228675, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3029464', 'n_tokens_mistral': 899, 'n_tokens_neox': 886, 'n_words': 231}