<commit_before>var aws = require('aws-sdk')
var s3 = new aws.S3({apiVersion: '2006-03-01'})

var _ = require('lodash')
  , async = require('async')
  , request = require('request')
  , Transform = require('stream').Transform
  , csv = require('csv-streamify')
  , JSONStream = require('JSONStream')

// Set LOGGLY_TOKEN to your Loggly customer token. It will look somethign like this:
// LOGGLY_TOKEN = '<PASSWORD>'
// Preferably, you should set the tag 'loggly-customer-tag' on the S3 bucket.


// Optionally set a LOGGLY_TAG if you want to tag these logs in a certain way. For example:
// LOGGLY_TAG = 'aws-elb-logs'
// Preferably, you should set the 'loggly-tag' on the S3 bucket.

LOGGLY_URL_BASE = 'https://logs-01.loggly.com/bulk/'
BUCKET_LOGGLY_TOKEN_NAME = 'loggly-customer-token'
BUCKET_LOGGLY_TAG_NAME = 'loggly-tag'

// Used if no S3 bucket tag soesn't contain customer token.
// Note: You either need to specify a cutomer token in this script or via the S3 bucket tag else an error is logged.
DEFAULT_LOGGLY_URL = null

if ( typeof LOGGLY_TOKEN !== 'undefined' ) { 
    DEFAULT_LOGGLY_URL = LOGGLY_URL_BASE + LOGGLY_TOKEN;

    if ( typeof LOGGLY_TAG !== 'undefined' ) {
	DEFAULT_LOGGLY_URL += '/tag/' + LOGGLY_TAG;
    }
}

if ( DEFAULT_LOGGLY_URL ) console.log('Loading elb2loggly, default Loggly endpoint: ' + DEFAULT_LOGGLY_URL);
else console.log('Loading elb2loggly, NO default Loggly endpoint, must be set in bucket tag ' + BUCKET_LOGGLY_TOKEN_NAME );

// AWS logs contain the following fields: (Note: a couple are parsed from within the field.)
// http://docs.aws.amazon.com/ElasticLoadBalancing/latest/DeveloperGuide/access-log-collection.html
var COLUMNS = [
              'timestamp',
              'elb',
              'client_ip', 'client_port', // split from client
              'backend', 'backend_port',
              'request_processing_time',
              'backend_processing_time',
              'response_processing_time',
              'elb_status_code',
              'backend_status_code',
              'received_bytes',
              'sent_bytes',
              'request_method', // Split from request
              'request_url'     // Split from request
              ];

// Parse elb log into component parts.
var parse_s3_log = function(data, encoding, done) {

  if ( data.length == 12 ) {

      // Split clientip:port and backendip:port at index 2,3
      data.splice(3,1,data[3].split(':'))
      data.splice(2,1,data[2].split(':'))
      data = _.flatten(data)

      // Pull the method from the request.  (WTF on Amazon's decision to keep these as one string.)
      var url_mash = data.pop()

      var url_mash = url_mash.split(' ',2)

      data.push(url_mash[0],url_mash[1])

      if ( data.length == COLUMNS.length ) {
         log =  _.zipObject(COLUMNS, data)
        this.push(log);
      } else {
	  console.error('ELB log length ' + data.length + ' did not match COLUMNS length ' + COLUMNS.length)
      }
  }
  done();

};


exports.handler = function(event, context) {<commit_msg>added support ofr additional fields added by AWS<commit_after>var aws = require('aws-sdk')
var s3 = new aws.S3({apiVersion: '2006-03-01'})

var _ = require('lodash')
  , async = require('async')
  , request = require('request')
  , Transform = require('stream').Transform
  , csv = require('csv-streamify')
  , JSONStream = require('JSONStream')

// Set LOGGLY_TOKEN to your Loggly customer token. It will look somethign like this:
// LOGGLY_TOKEN = '<PASSWORD>'
// Preferably, you should set the tag 'loggly-customer-tag' on the S3 bucket.


// Optionally set a LOGGLY_TAG if you want to tag these logs in a certain way. For example:
// LOGGLY_TAG = 'aws-elb-logs'
// Preferably, you should set the 'loggly-tag' on the S3 bucket.

LOGGLY_URL_BASE = 'https://logs-01.loggly.com/bulk/'
BUCKET_LOGGLY_TOKEN_NAME = 'loggly-customer-token'
BUCKET_LOGGLY_TAG_NAME = 'loggly-tag'

// Used if no S3 bucket tag soesn't contain customer token.
// Note: You either need to specify a cutomer token in this script or via the S3 bucket tag else an error is logged.
DEFAULT_LOGGLY_URL = null

if ( typeof LOGGLY_TOKEN !== 'undefined' ) { 
    DEFAULT_LOGGLY_URL = LOGGLY_URL_BASE + LOGGLY_TOKEN;

    if ( typeof LOGGLY_TAG !== 'undefined' ) {
	DEFAULT_LOGGLY_URL += '/tag/' + LOGGLY_TAG;
    }
}

if ( DEFAULT_LOGGLY_URL ) console.log('Loading elb2loggly, default Loggly endpoint: ' + DEFAULT_LOGGLY_URL);
else console.log('Loading elb2loggly, NO default Loggly endpoint, must be set in bucket tag ' + BUCKET_LOGGLY_TOKEN_NAME );

// AWS logs contain the following fields: (Note: a couple are parsed from within the field.)
// http://docs.aws.amazon.com/ElasticLoadBalancing/latest/DeveloperGuide/access-log-collection.html
var COLUMNS = [
              'timestamp',
              'elb',
              'client_ip', 'client_port', // split from client
              'backend', 'backend_port',
              'request_processing_time',
              'backend_processing_time',
              'response_processing_time',
              'elb_status_code',
              'backend_status_code',
              'received_bytes',
              'sent_bytes',
              'request_method', // Split from request
              'request_url',     // Split from request
              'user_agent',
              'ssl_cipher',
              'ssl_protocol'
              ];

// Parse elb log into component parts.
var parse_s3_log = function(data, encoding, done) {
  var expectedFields = 15;
  if ( data.length == expectedFields ) {

      // Split clientip:port and backendip:port at index 2,3
      data.splice(3,1,data[3].split(':'))
      data.splice(2,1,data[2].split(':'))
      
      // Pull the method from the request.  (WTF on Amazon's decision to keep these as one string.)
      var requestIndex = data.length - 4;
      var url_mash = data[requestIndex];

      url_mash = url_mash.split(' ',2)

      data.splice(requestIndex,1, url_mash);
      data = _.flatten(data)

      if ( data.length == COLUMNS.length ) {
        this.push(_.zipObject(COLUMNS, data));
      } else {
	      console.error('ELB log length ' + data.length + ' did not match COLUMNS length ' + COLUMNS.length)
      }
      done();
  }
  else {
    done("expecting " + expectedFields + " actual fields " + data.length);
  }

};


exports.handler = function(event, context) {
{'fraction_non_alphanumeric': 0.11907750235331033, 'fraction_numerical': 0.01004079071226859, 'mean_word_length': 2.828828828828829, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6291179', 'n_tokens_mistral': 2070, 'n_tokens_neox': 2027, 'n_words': 551}