<commit_before>var ms = require('ms')
var bot = require('./index.js')

var src = process.env.PIPERMAIL_SOURCE || 'https://mail.mozilla.org/pipermail/es-discuss/'
var db = process.env.PIPERMAIL_DATABASE

var didSomething = new Date()

var settings = 'last-reboot:  ' + (new Date()).toISOString() + '\n' +
               'source:       ' + src + '\n' +
               'database:     ' + (db || 'no database attached').replace(/^.*@/,'')
var lastRun = 'no old runs to display'
var lastStart = 'never started'
var lastEnd = 'never finished'
var lastMessage = 'no messages processsed yet'

function run() {
  if (lastEnd != 'never finished') {
    lastRun = lastStart + ' to ' + lastEnd
  }
  lastStart = (new Date()).toISOString()
  var defaultMonths = (new Date()).getDate() < 5 ? 2 : 1
  return bot({source: src, db: db, months: +(process.env.PIPERMAIL_MONTHS || defaultMonths), parallel: +(process.env.PIPERMAIL_PARALLEL || 1)})
    .on('data', function (message) {
      lastMessage = message.id
      didSomething = new Date()
    })
    .wait()
    .then(function () {
      didSomething = new Date()
      lastEnd = (new Date()).toISOString()
    })
}
maintain()
function maintain() {
  run().done(maintain, maintain)
}

var http = require('http')
<commit_msg>Log the start and end time of each run.<commit_after>var ms = require('ms')
var bot = require('./index.js')

var src = process.env.PIPERMAIL_SOURCE || 'https://mail.mozilla.org/pipermail/es-discuss/'
var db = process.env.PIPERMAIL_DATABASE
var logConnection
if (db) {
  logConnection = mongojs(db, ['log'])
}

var didSomething = new Date()

var settings = 'last-reboot:  ' + (new Date()).toISOString() + '\n' +
               'source:       ' + src + '\n' +
               'database:     ' + (db || 'no database attached').replace(/^.*@/,'')
var lastRun = 'no old runs to display'
var lastStart = 'never started'
var lastEnd = 'never finished'
var lastMessage = 'no messages processsed yet'

function run() {
  if (lastEnd != 'never finished') {
    lastRun = lastStart + ' to ' + lastEnd
  }
  lastStart = (new Date()).toISOString()
  var defaultMonths = (new Date()).getDate() < 5 ? 2 : 1
  return bot({source: src, db: db, months: +(process.env.PIPERMAIL_MONTHS || defaultMonths), parallel: +(process.env.PIPERMAIL_PARALLEL || 1)})
    .on('data', function (message) {
      lastMessage = message.id
      didSomething = new Date()
    })
    .wait()
    .then(function () {
      didSomething = new Date()
      lastEnd = (new Date()).toISOString()
      if (logConnection) {
        logConnection.log.insert({
          type: 'bot-run',
          start: new Date(lastStart),
          end: new Date(lastEnd)
        }, {safe: true}, function (err) {
          if (err) {
            console.error(err.stack || err.message || err)
          }
        })
      }
    })
}
maintain()
function maintain() {
  run().done(maintain, maintain)
}

var http = require('http')

{'fraction_non_alphanumeric': 0.16462167689161555, 'fraction_numerical': 0.0027266530334014998, 'mean_word_length': 2.897742363877822, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2785193', 'n_tokens_mistral': 972, 'n_tokens_neox': 961, 'n_words': 249}