<commit_before>var AWS = require('aws-sdk');
var _ = require('lodash');
var base64 = require('base-64');

exports.index = function(event, context, callback) {
  var recordCount = event.Records.length;
  console.log(recordCount);

  for (i = 0; i < recordCount; i++) {
    var kinesis = event.Records[i].kinesis;
    data = base64.decode(kinesis.data);
    console.log(data);
  }


  callback(null, 'Hello from Lambda');
}<commit_msg>lambda function cleans up the payload provided by kinesis<commit_after>var AWS = require('aws-sdk');
var _ = require('lodash');
var base64 = require('base-64');

/**
 * cleanJSONString
 *
 * @param {any} string string that requires cleaning
 * @returns string cleaned string
 */
function cleanJSONString(string) {
  // preserve newlines, etc - use valid JSON
  string = string.replace(/\\n/g, '\\n')
               .replace(/\\'/g, '\\\'')
               .replace(/\\"/g, '\\"')
               .replace(/\\&/g, '\\&')
               .replace(/\\r/g, '\\r')
               .replace(/\\t/g, '\\t')
               .replace(/\\b/g, '\\b')
               .replace(/\\f/g, '\\f');
  // remove non-printable and other non-valid JSON chars
  return string.replace(/[\u0000-\u0019]+/g, '');
}

/**
 * Main lambda handler and entry point
 */
exports.index = function(event, context, callback) {
  // find the number of records being given to us by kinesis
  var recordCount = event.Records.length;

  // iterate through the records
  for (i = 0; i < recordCount; i++) {
    // get the kinesis payload
    var kinesis = event.Records[i].kinesis;

    // convert from base64, clean up and "funny" characters and parse into JSON
    data = JSON.parse(cleanJSONString(base64.decode(kinesis.data)));

    console.log(data);
  }


  callback(null, 'Hello from Lambda');
}
{'fraction_non_alphanumeric': 0.17058165548098433, 'fraction_numerical': 0.013422818791946308, 'mean_word_length': 2.6510204081632653, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5106650', 'n_tokens_mistral': 597, 'n_tokens_neox': 565, 'n_words': 151}