<commit_before>var https = require('https');
var qs = require('querystring');
var AWS = require('aws-sdk');
var s3 = new AWS.S3();

var config = {
  untappdUrl: 'https://api.untappd.com/v4/thepub/local',
  untappdClientId: '',
  untappdClientSecret: '',
  bucketName: ''
};

exports.handler = function(event, context) {
  /*console.log('Received request:', JSON.stringify(event, null, 2));*/

  var httpQueryParams = {
    client_id: config.untappdClientId,
    client_secret: config.untappdClientSecret,
    lat: 61.498171,
    lng: 23.762101,
    dist_pref: 'km',
    radius: 2
  };
  var httpOptions = {
    host: 'api.untappd.com',
    path: '/v4/thepub/local'+'?'+qs.stringify(httpQueryParams)
  };

  https.get(httpOptions, function(res) {
    if (res.statusCode != 200) {
      context.fail('Received non-ok status code from Untappd: ', res.statusCode);
    }

    var body = '';
    res.on('data', function(chunk) { body += chunk; });
    res.on('end', function() {

      var s3params = {
        'Bucket': config.bucketName,
        'Key': 'data/thepub.json',
        'ContentType': 'application/json',
        'Body': body
      };

      s3.putObject(s3params, function(err, data) {
        if (err) {
          console.log(err, err.stack);
          context.fail('Error while uploading to S3');
        }
        else {
          context.succeed('Successfully updated data');
        }
      });
    });

  }).on('error', function(e) {
    console.log(e.message);
    context.fail('Failed to get data from Untappd');
  });

};<commit_msg>Use streaming upload instead of buffering whole response in lambda<commit_after>var https = require('https');
var qs = require('querystring');
var AWS = require('aws-sdk');
var s3 = new AWS.S3();

var config = {
  untappdUrl: 'https://api.untappd.com/v4/thepub/local',
  untappdClientId: '',
  untappdClientSecret: '',
  bucketName: ''
};

exports.handler = function(event, context) {

  var httpQueryParams = {
    client_id: config.untappdClientId,
    client_secret: config.untappdClientSecret,
    lat: 61.498171,
    lng: 23.762101,
    dist_pref: 'km',
    radius: 2
  };
  var httpOptions = {
    host: 'api.untappd.com',
    path: '/v4/thepub/local'+'?'+qs.stringify(httpQueryParams)
  };

  https.get(httpOptions, function(res) {
    if (res.statusCode != 200) {
      context.fail('Received non-ok status code from Untappd: ', res.statusCode);
    }

    var s3params = {
      'Bucket': config.bucketName,
      'Key': 'data/thepub.json',
      'ContentLength': res.headers['content-length'],
      'ContentType': 'application/json',
      'Body': res
    };

    s3.upload(s3params, function(err, data) {
      if (err) {
        console.log(err, err.stack);
        context.fail('Error while uploading to S3');
      }
      else {
        context.succeed('Successfully updated data');
      }
    });

  }).on('error', function(e) {
    console.log(e.message);
    context.fail('Failed to get data from Untappd');
  });

};
{'fraction_non_alphanumeric': 0.16265060240963855, 'fraction_numerical': 0.019076305220883535, 'mean_word_length': 2.9174311926605503, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3269726', 'n_tokens_mistral': 1054, 'n_tokens_neox': 1019, 'n_words': 204}