<commit_before>/* jshint esnext: true, node: true */
'use strict';

var util = require('util');
var fs = require('fs');
var path = require('path');

var log = require('loglevel');
var _ = require('lodash');
var moment = require('moment');
var AWS = require('aws-sdk');
var progress = require('progress-stream');
var prettyBytes = require('pretty-bytes');
var numeral = require('numeral');
var child_process = require('child_process');
var zlib = require('zlib');
var debounce = require('debounce');



module.exports = DBR;

function DBR(credentials, stagingCredentials, bucket, stagingBucket) {
  this.credentials = credentials;
  this.stagingCredentials = stagingCredentials;
  this.bucket = bucket;
  this.stagingBucket = stagingBucket;

  this.dbrClient = new AWS.S3(this.credentials);
  this.stagingClient = new AWS.S3(this.stagingCredentials);
}


// Download, unzip, gzip, upload a DBR to the staging bucket
// Returns a promise that resolves when everything is finished.
DBR.prototype.stageDBR = function(month) {
  let self = this;
  return this.findDBR(month)
    .then(function(result) {
      return downloadDBR(result, self.dbrClient, self.bucket);
    })
    .then(function(result) {
      return processDBR(result, self.stagingClient, self.stagingBucket);
    });
};


// Find a DBR for a given month or raise an error
// Argument is a UTC moment object representing midnight on the first of the
// desired month.
DBR.prototype.findDBR = function(month) {
  let self = this;
  return new Promise(function (resolve, reject) {
    self.getDBRs()
        .then(function(dbrs) {
          let match = _.find(dbrs, function(d) { return month.isSame(d.Date); });
          if (match === undefined) {
            return reject(new Error(`Unable to find the DBR for ${month.format('MMMM YYYY')}.`));
          } else {
            return resolve(match);
          }
        });
  });
};

// Find a staged DBR for a given month or raise an error
// Argument is a UTC moment object representing midnight on the first of the
// desired month.
// TODO: is there a way to consolidate the handler in then() with findDBR()'s?
DBR.prototype.findStagedDBR = function(month) {
  let self = this;
  return new Promise(function (resolve, reject) {
    self.getStagedDBRs()
        .then(function(dbrs) {
          let match = _.find(dbrs, function(d) { return month.isSame(d.Date); });
          if (match === undefined) {
            return reject(new Error(`Unable to find the staged DBR for ${month.format('MMMM YYYY')}.`));
          } else {
            return resolve(match);
          }
        });
  });
};

// Get the contents of a bucket. Returns a promise which resolves with an array
// of bucket objects.
// Will not work with buckets containing > 1000 objects, but that's okay
// for our purposes here.
DBR.prototype.getBucketContents = function(client, bucket) {
  return new Promise(function (resolve, reject) {
    client.listObjects({Bucket: bucket}, function(err, data) {
      if (err) return reject(err);
      if ('Contents' in data) {
        return resolve(data.Contents);
      } else {
        return reject(`Bucket listObjects response didn't contain "Contents" key.`);
      }
    });
  });<commit_msg>Improve documentation of DBR(month) methods<commit_after>/* jshint esnext: true, node: true */
'use strict';

var util = require('util');
var fs = require('fs');
var path = require('path');

var log = require('loglevel');
var _ = require('lodash');
var moment = require('moment');
var AWS = require('aws-sdk');
var progress = require('progress-stream');
var prettyBytes = require('pretty-bytes');
var numeral = require('numeral');
var child_process = require('child_process');
var zlib = require('zlib');
var debounce = require('debounce');



module.exports = DBR;

function DBR(credentials, stagingCredentials, bucket, stagingBucket) {
  this.credentials = credentials;
  this.stagingCredentials = stagingCredentials;
  this.bucket = bucket;
  this.stagingBucket = stagingBucket;

  this.dbrClient = new AWS.S3(this.credentials);
  this.stagingClient = new AWS.S3(this.stagingCredentials);
}


// Download, unzip, gzip, upload a DBR to the staging bucket
// Returns a promise that resolves when everything is finished.
// Month is a UTC moment object for midnight on the first of the month.
DBR.prototype.stageDBR = function(month) {
  let self = this;
  return this.findDBR(month)
    .then(function(result) {
      return downloadDBR(result, self.dbrClient, self.bucket);
    })
    .then(function(result) {
      return processDBR(result, self.stagingClient, self.stagingBucket);
    });
};


// Find a DBR for a given month or raise an error
// Month is a UTC moment object for midnight on the first of the month.
DBR.prototype.findDBR = function(month) {
  let self = this;
  return new Promise(function (resolve, reject) {
    self.getDBRs()
        .then(function(dbrs) {
          let match = _.find(dbrs, function(d) { return month.isSame(d.Date); });
          if (match === undefined) {
            return reject(new Error(`Unable to find the DBR for ${month.format('MMMM YYYY')}.`));
          } else {
            return resolve(match);
          }
        });
  });
};

// Find a staged DBR for a given month or raise an error
// Month is a UTC moment object for midnight on the first of the month.
// TODO: is there a way to consolidate the handler in then() with findDBR()'s?
DBR.prototype.findStagedDBR = function(month) {
  let self = this;
  return new Promise(function (resolve, reject) {
    self.getStagedDBRs()
        .then(function(dbrs) {
          let match = _.find(dbrs, function(d) { return month.isSame(d.Date); });
          if (match === undefined) {
            return reject(new Error(`Unable to find the staged DBR for ${month.format('MMMM YYYY')}.`));
          } else {
            return resolve(match);
          }
        });
  });
};

// Get the contents of a bucket. Returns a promise which resolves with an array
// of bucket objects.
// Will not work with buckets containing > 1000 objects, but that's okay
// for our purposes here.
DBR.prototype.getBucketContents = function(client, bucket) {
  return new Promise(function (resolve, reject) {
    client.listObjects({Bucket: bucket}, function(err, data) {
      if (err) return reject(err);
      if ('Contents' in data) {
        return resolve(data.Contents);
      } else {
        return reject(`Bucket listObjects response didn't contain "Contents" key.`);
      }
    });
  });
{'fraction_non_alphanumeric': 0.1277282508453735, 'fraction_numerical': 0.0018444512757454657, 'mean_word_length': 3.2557226945716153, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1584724', 'n_tokens_mistral': 1940, 'n_tokens_neox': 1912, 'n_words': 618}