<commit_before>"use strict";
var tracker = require("gdq_tracker_scraper")

var SUPER_METROID_BID_ID = 4792;

function getChoiceSeries(bid) {
  return new Promise(function(resolve, reject) {
      tracker.getBidDetail(bid).then(function(res){
      var promises = []
      res.data.forEach(function(d){
        promises.push(tracker.getBidDetail(d.link.split('/').pop()));
      });
      Promise.all(promises).then(function(values){
        resolve(values)
      })
    });
  });
}

function processChoiceSeries(data) {
  var xArrs = data.map(function(d){return d.data.map(function(e){return e.date})})
  xArrs.forEach(function(arr, idx){
    arr.unshift('x' + idx);
  }); 
  var values = data.map(function(d){return d.data.map(function(e){return e.amount})})
  values.forEach(function(arr, idx){
    arr.unshift(data[idx].bid)
  });
  var payload = {
    xs: xArrs,
    ys: values
  }
  return payload;
}

getChoiceSeries(SUPER_METROID_BID_ID).then(function(data){
  var series = processChoiceSeries(data)
  console.log(series)
})<commit_msg>Aggregation and google storage upload<commit_after>"use strict";
var tracker = require("gdq_tracker_scraper");
var gcloud = require('gcloud')({
  projectId: 'sgdq-backend',
  keyFilename: './credentials.json'
});

var SUPER_METROID_BID_ID = 4792;

function getChoiceSeries(bid) {
  return new Promise(function(resolve, reject) {
      tracker.getBidDetail(bid).then(function(res){
      var promises = []
      res.data.forEach(function(d){
        promises.push(tracker.getBidDetail(d.link.split('/').pop()));
      });
      Promise.all(promises).then(function(values){
        resolve(values)
      })
    });
  });
}

function processChoiceSeries(data) {
  var xArrs = data.map(function(d){return d.data.map(function(e){return e.date})})
  var end = xArrs.reduce(function(prev, curr) { return curr[0] < prev ? curr[0] : prev}, Infinity);
  var start   = xArrs.reduce(function(prev, curr) { return curr[curr.length - 1] > prev ? curr[curr.length - 1] : prev }, 0)
  var windowLength = 1000 * 60 * 10 // 10 min windows
  var trueX = []
  for(var i = start; i < end; i+= windowLength) trueX.push(i)
  var values = data.map(function(d){return d.data.map(function(e){return e.amount})})
  var aggregatedValues = [];

  values.forEach(function(series, seriesIdx) {
    var aggregatedSeries = []
    trueX.forEach(function(windowStart, idx){
      aggregatedSeries.push(series.reduce(function(prev, curr, idx){
        if(xArrs[seriesIdx][idx] >= windowStart && xArrs[seriesIdx][idx] < windowStart + windowLength){
          prev += curr;
        }
        return prev; 
      }, 0))
    })
    aggregatedValues.push(aggregatedSeries)
  });
  trueX.unshift('x');
  aggregatedValues.forEach(function(series, idx){
    series.unshift(data[idx].bid)
  })
  console.log(aggregatedValues)
  var payload = {
    x: trueX,
    ys: aggregatedValues
  }
  return payload;
}

var bucket = gcloud.storage().bucket('sgdq-backend.appspot.com');

getChoiceSeries(SUPER_METROID_BID_ID).then(function(data){
  var series = processChoiceSeries(data)
  var file = bucket.file('killVsSave.json');
  file.save(JSON.stringify(series), function(err){
    if(err) console.log(err)
    else {
      console.log('Upload successful')
      file.makePublic();
    }
  })
});
{'fraction_non_alphanumeric': 0.1569820505019775, 'fraction_numerical': 0.007301490721022209, 'mean_word_length': 3.878338278931751, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5791518', 'n_tokens_mistral': 1148, 'n_tokens_neox': 1116, 'n_words': 190}