<commit_before>/*
 * SANS ISC 404Project v1
 *
 * http://isc.sans.edu/tools/404project.html
 *
 * Purpose: Collect 404 Page Not Found to compile
 *          list for potential crawler trends
 *
 * Instructions: Enter the following from https://isc.sans.edu/myinfo.html
 *                for the variables below
 *     $sMyUserID - in right-column under "Logged in as:"
 *     $sMyKey - in section https://isc.sans.edu/myinfo.html#report
 *               labeled "Your Authentication Key:"
 */

var https = require('https');

function formatYmd(ts) {
  var year = ts.getUTCFullYear();
  var month = ts.getUTCMonth() + 1;
  var day = ts.getUTCDate();

  return year + "-" + month + "-" + day;
}

function formatHms(ts) {
  var hour = ts.getUTCHours();
  var minute = ts.getUTCMinutes() + 1;
  var second = ts.getUTCSeconds();

  return hour + ":" + minute + ":" + second;
}

function Reporter(userID, key) {
  this.userID = userID;
  this.key = key;
}

Reporter.prototype.report404 = function() {
  var url;
  var ip;
  var ua;
  var ts;

  if (arguments.length == 1) {
    url = request.url;
    ip = request.connection.remoteAddress;
    ua = request.headers['user-agent'];
    ts = new Date();
  }
  else {
    url = arguments[0];
    ip = arguments[1];
    ua = arguments[2];
    ts = arguments[3] || new Date();
  }

  var req = https.request({
    host: 'isc.sans.edu',
    port: 443,
    path: '/weblogs/404project.html?id=' + this.userID + '&version=2',
    method: 'POST',
    headers: {
      'Content-Type' : 'application/x-www-form-urlencoded'
    }
  }, function(res) {
    // console.log('STATUS: ' + res.statusCode);
    // console.log('HEADERS: ' + JSON.stringify(res.headers));
    res.setEncoding('utf8');
    res.on('data', function (chunk) {
        //console.log('BODY: ' + chunk);
    });
  });

  req.on('error', function(e) {
    // console.log('problem with request: ' + e.message);
  });

  var data = this.userID + '\0' + this.key + '\0' + url + '\0' + ip + '\0' + ua + '\0' + formatYmd(ts) + '\0' + formatHms(ts);
  var buf = new Buffer(data);
  var payload = buf.toString('base64');

  // console.log("data=%s", data);
  // console.log("paylaod=%s", payload);

  req.write('DATA=' + payload);
  req.end();
};

module.exports.create = function(userID, key) {
    return new Reporter(userID, key);
};<commit_msg>code cleanup.<commit_after>/*
 * SANS ISC 404Project v1
 *
 * http://isc.sans.edu/tools/404project.html
 *
 * Purpose: Collect 404 Page Not Found to compile
 *          list for potential crawler trends
 *
 * Instructions: Enter the following from https://isc.sans.edu/myinfo.html
 *                for the variables below
 *     $sMyUserID - in right-column under "Logged in as:"
 *     $sMyKey - in section https://isc.sans.edu/myinfo.html#report
 *               labeled "Your Authentication Key:"
 */
 /*jshint  devel:true, node:true, indent:2, maxerr:50 */
(function () {
  "use strict";
  var https = require("https");
  function formatYmd(ts) {
    var year = ts.getUTCFullYear();
    var month = ts.getUTCMonth() + 1;
    var day = ts.getUTCDate();
    return year + "-" + month + "-" + day;
  }
  function formatHms(ts) {
    var hour = ts.getUTCHours();
    var minute = ts.getUTCMinutes() + 1;
    var second = ts.getUTCSeconds();
    return hour + ":" + minute + ":" + second;
  }
  function Reporter(userID, key) {
    this.userID = userID;
    this.key = key;
  }
  Reporter.prototype.report404 = function () {
    var url;
    var ip;
    var ua;
    var ts;
    if (arguments.length === 1) {
      var request = arguments[0];
      url = request.url;
      ip = request.connection.remoteAddress;
      ua = request.headers["user-agent"];
      ts = new Date();
    } else {
      url = arguments[0];
      ip = arguments[1];
      ua = arguments[2];
      ts = arguments[3] || new Date();
    }
    var req = https.request({
      host: "isc.sans.edu",
      port: 443,
      path: "/weblogs/404project.html?id=" + this.userID + "&version=2",
      method: "POST",
      headers: {
        "Content-Type": "application/x-www-form-urlencoded"
      }
    }, function (res) {
      res.setEncoding("utf8");
      res.on("data", function (chunk) {
        console.log(chunk);
      });
    });
    var data = this.userID + "\x00" + this.key + "\x00" + url + "\x00" + ip +
               "\x00" + ua + "\x00" + formatYmd(ts) + "\x00" + formatHms(ts);
    var buf = new Buffer(data);
    var payload = buf.toString("base64");
    req.write("DATA=" + payload);
    req.end();
  };
  module.exports.create = function (userID, key) {
    return new Reporter(userID, key);
  };
})();
{'fraction_non_alphanumeric': 0.1703047330883942, 'fraction_numerical': 0.017722066133563863, 'mean_word_length': 2.56, 'pattern_counts': {'":': 5, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3662469', 'n_tokens_mistral': 1651, 'n_tokens_neox': 1586, 'n_words': 375}