<commit_before>var AWS = require('aws-sdk');
var split = require('split');
var util = require('util');
var winston = require('winston');
require('winston-papertrail').Papertrail;
var zlib = require('zlib');

var format = require('./formats/cloudfront');

// get reference to S3 client
var s3 = new AWS.S3({ region: 'us-west-2' });

// data - json object representation of a log line.
var _reformat = function(data) {
  row = [];
  for (var key in data) {
    row.push(util.format('%s="%s"', key, data[key]));
  }
  return row.join(' ');
}

var _processEvent = function(logger, event, callback) {
  // Read options from the event.
  console.log("Reading options from event:\n", util.inspect(event, {depth: 5}));
  var srcBucket = event.Records[0].s3.bucket.name;
  // Object key may have spaces or unicode non-ASCII characters.
  var srcKey = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, " "));

  console.log('Processing', srcKey);

  var params = {
    Bucket: srcBucket,
    Key: srcKey
  };

  var read = s3.getObject(params).createReadStream();
  var reader = read;
  if (format.gzip) {
    var gunzip = zlib.createGunzip();
    read.pipe(gunzip);
    reader = gunzip;
  }

  var stream = reader.pipe(split()); // split() makes each line in the stream a processable chunk.
  stream.on('data', function(row) {
    var data = format.toJson(row);
    if (data) {
      logger.log('info', _reformat(data));
    }
  });

  stream.on('error', function(err) {
    callback(err);
  });

  stream.on('end', function() {
    callback();<commit_msg>extract region from event<commit_after>var AWS = require('aws-sdk');
var split = require('split');
var util = require('util');
var winston = require('winston');
require('winston-papertrail').Papertrail;
var zlib = require('zlib');

var format = require('./formats/cloudfront');

// data - json object representation of a log line.
var _reformat = function(data) {
  var row = [];
  for (var key in data) {
    row.push(util.format('%s="%s"', key, data[key]));
  }
  return row.join(' ');
}

var _processEvent = function(logger, event, callback) {
  // Read options from the event.
  console.log("Reading options from event:\n", util.inspect(event, {depth: 5}));
  var record = event.Records[0];
  var srcRegion = record.awsRegion;
  var srcBucket = record.s3.bucket.name;
  // Object key may have spaces or unicode non-ASCII characters.
  var srcKey = decodeURIComponent(record.s3.object.key.replace(/\+/g, " "));

  console.log('Processing', srcKey);

  var params = {
    Bucket: srcBucket,
    Key: srcKey
  };

  var s3 = new AWS.S3({ region: srcRegion });  // our s3 client
  var read = s3.getObject(params).createReadStream();
  var reader = read;
  if (format.gzip) {
    var gunzip = zlib.createGunzip();
    read.pipe(gunzip);
    reader = gunzip;
  }

  var stream = reader.pipe(split()); // split() makes each line in the stream a processable chunk.
  stream.on('data', function(row) {
    var data = format.toJson(row);
    if (data) {
      logger.log('info', _reformat(data));
    }
  });

  stream.on('error', function(err) {
    callback(err);
  });

  stream.on('end', function() {
    callback();
{'fraction_non_alphanumeric': 0.1681360201511335, 'fraction_numerical': 0.005667506297229219, 'mean_word_length': 3.597684515195369, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5839360', 'n_tokens_mistral': 1076, 'n_tokens_neox': 1049, 'n_words': 275}