<commit_before>var config = require('./config.js');

var fs = require('fs');
var parse = require('clf-parser');
var glob = require('glob');
var path = require('path');
var _ = require('lodash');
var Tail = require('always-tail');
var express = require('express');
var exec = require('child_process').exec;
var request = require('request');

var interval = config.interval;
var apikey = config.apikey;
var reportTo = config.reportTo;

var dir = config.dir;
if (!dir) {
  // If we see nginx monitor that, if not monitor Apache.
  // We always use nginx as a reverse proxy to node apps and
  // to mixed node/apache servers, so it should catch all if present
  dir = '/var/log/nginx';
  if (!fs.existsSync(dir)) {
    dir = '/var/log/apache2';
  }
}

var files = [];

files = files.concat(glob.sync(dir + '/*log'));

var scoreboard = {};

var sites = [];

_.each(files, function(file) {
  addFile(file);
});

function addFile(file) {
  if (file.indexOf('error') !== -1) {
    return;
  }
  if (_.some(sites, function(site) {
    return site.file === file;
  })) {
    // Already watching
    return;
  }
  var name = path.basename(file);
  name = name.replace(/[\._\-]log/, '');
  name = name.replace(/[\._\-]access/, '');
  if (name === 'access') {
    return;
  }
  sites.push({ 
    name: name, 
    file: file, 
    stats: {
      name: name, 
      interval: interval
    }
  });
}

function removeFile(file) {
  var site = _.find(sites, function(site) {
    return site.file === file;
  });
  if (site) {
    site.shutdown();
    sites = _.filter(sites, function(site) {
      return site.file !== file;
    });
  }
}

_.each(sites, function(site) {
  watch(site);
});

fs.watch(dir, function(event, filename) {
  filename = dir + '/' + filename;
  // Added? Removed? Merely appended to?
  // 'rename' and 'change' are pretty incomplete as events
  // go, folks. Let's just figure it out
  if (fs.existsSync(filename)) {
    addFile(filename);
  } else {
    removeFile(filename);
  }
});

setInterval(step, interval);
step();

function step() {
  return serverLevelStats(function() {
    _.each(sites, function(site) {
      if (site.flush) {
        site.flush();
      }
    });
    report();
  });
}

function report() {
  scoreboard.sites = _.map(sites, function(site) {
    return site.stats;
  });
  scoreboard.apikey = apikey;
  request({
    method: 'POST',
    uri: reportTo,
    json: scoreboard
  }, function(err, response, body) {
    if (err) {
      console.error(err);
      // TODO: get the admin's attention in some other way if we cannot talk to the scoreboard server for
      // a long time
    }
  });
}

function watch(site) {
  var size = fs.statSync(site.file).size;
  var tail = new Tail(site.file, "\n", { start: size });
  var now = getEpoch(new Date());
  var pages = 0;
  var errors = 0;
  var errorDetails = [];
  tail.on('line', function(line) {
    try {
      info = parse(line);
      if (error(info)) {
        errors++;
        if (errorDetails.length < 10) {
          errorDetails.push(info);
        }
      }
      if (page(info)) {
        pages++;
      }
    } catch (e) {
      throw e;
      // Don't fuss if a bad line is encountered
    }
    console.log(errorDetails);
  });

  site.shutdown = function() {
    tail.unwatch();
  };

  site.flush = function() {
    site.stats = {
      name: site.name,
      pages: pages,
      errors: errors,
      errorDetails: errorDetails,
      interval: interval
    };
    now++;
    errors = 0;
    errorDetails = [];
    pages = 0;
  };
}

function getEpoch(date) {
  return date.getTime() / interval;
}

// We are interested in server errors, not user errors
function error(info) {
  return (info.status >= 500);
}
<commit_msg>.gz files can sneak in if it shows up as a rename event in the filesystem. We don't look at those, we look at the current live access.log file<commit_after>var config = require('./config.js');

var fs = require('fs');
var parse = require('clf-parser');
var glob = require('glob');
var path = require('path');
var _ = require('lodash');
var Tail = require('always-tail');
var express = require('express');
var exec = require('child_process').exec;
var request = require('request');

var interval = config.interval;
var apikey = config.apikey;
var reportTo = config.reportTo;

var dir = config.dir;
if (!dir) {
  // If we see nginx monitor that, if not monitor Apache.
  // We always use nginx as a reverse proxy to node apps and
  // to mixed node/apache servers, so it should catch all if present
  dir = '/var/log/nginx';
  if (!fs.existsSync(dir)) {
    dir = '/var/log/apache2';
  }
}

var files = [];

files = files.concat(glob.sync(dir + '/*log'));

var scoreboard = {};

var sites = [];

_.each(files, function(file) {
  addFile(file);
});

function addFile(file) {
  if (file.indexOf('error') !== -1) {
    return;
  }
  if (_.some(sites, function(site) {
    return site.file === file;
  })) {
    // Already watching
    return;
  }
  var name = path.basename(file);
  name = name.replace(/[\._\-]log/, '');
  name = name.replace(/[\._\-]access/, '');
  if (name === 'access') {
    return;
  }
  sites.push({ 
    name: name, 
    file: file, 
    stats: {
      name: name, 
      interval: interval
    }
  });
}

function removeFile(file) {
  var site = _.find(sites, function(site) {
    return site.file === file;
  });
  if (site) {
    site.shutdown();
    sites = _.filter(sites, function(site) {
      return site.file !== file;
    });
  }
}

_.each(sites, function(site) {
  watch(site);
});

fs.watch(dir, function(event, filename) {
  filename = dir + '/' + filename;
  // Added? Removed? Merely appended to?
  // 'rename' and 'change' are pretty incomplete as events
  // go, folks. Let's just figure it out

  // If a file has just been compressed and rotated out in favor
  // of a new log file, the filename will change so the extension
  // is no longer .log. If this happens drop the file. A new file
  // with the right name will appear and take over

  if (!filename.match(/log$/)) {
    removeFile(filename);
  } else if (fs.existsSync(filename)) {
    addFile(filename);
  } else {
    removeFile(filename);
  }
});

setInterval(step, interval);
step();

function step() {
  return serverLevelStats(function() {
    _.each(sites, function(site) {
      if (site.flush) {
        site.flush();
      }
    });
    report();
  });
}

function report() {
  scoreboard.sites = _.map(sites, function(site) {
    return site.stats;
  });
  scoreboard.apikey = apikey;
  request({
    method: 'POST',
    uri: reportTo,
    json: scoreboard
  }, function(err, response, body) {
    if (err) {
      console.error(err);
      // TODO: get the admin's attention in some other way if we cannot talk to the scoreboard server for
      // a long time
    }
  });
}

function watch(site) {
  var size = fs.statSync(site.file).size;
  var tail = new Tail(site.file, "\n", { start: size });
  var now = getEpoch(new Date());
  var pages = 0;
  var errors = 0;
  var errorDetails = [];
  tail.on('line', function(line) {
    try {
      info = parse(line);
      if (error(info)) {
        errors++;
        if (errorDetails.length < 10) {
          errorDetails.push(info);
        }
      }
      if (page(info)) {
        pages++;
      }
    } catch (e) {
      throw e;
      // Don't fuss if a bad line is encountered
    }
  });

  site.shutdown = function() {
    tail.unwatch();
  };

  site.flush = function() {
    site.stats = {
      name: site.name,
      pages: pages,
      errors: errors,
      errorDetails: errorDetails,
      interval: interval
    };
    now++;
    errors = 0;
    errorDetails = [];
    pages = 0;
  };
}

function getEpoch(date) {
  return date.getTime() / interval;
}

// We are interested in server errors, not user errors
function error(info) {
  return (info.status >= 500);
}

{'fraction_non_alphanumeric': 0.1588654286441109, 'fraction_numerical': 0.0027982701602645636, 'mean_word_length': 2.63691026827012, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6145026', 'n_tokens_mistral': 2639, 'n_tokens_neox': 2567, 'n_words': 741}