<commit_before>/* jshint esnext: true, node: true */
'use strict';

/*******************************************************************************
Import month-to-date DBRs, overwriting the existing month-to-date.



*******************************************************************************/



var util = require('util');
var path = require('path');

var log = require('loglevel');
var ArgumentParser = require('argparse').ArgumentParser;
var rollbar = require('rollbar');
var moment = require('moment');

var BaseParser = require('./lib/baseparser.js');
var DBR = require('./lib/dbr.js');
var Redshift = require('./lib/redshift.js');
var cliUtils = require('./lib/cliutils.js');

rollbar.init(process.env.ROLLBAR_TOKEN, {environment: process.env.ROLLBAR_ENVIRONMENT});
rollbar.handleUncaughtExceptions(process.env.ROLLBAR_TOKEN,
                                 {exitOnUncaughtException: true});


var parser = new BaseParser({
  version: '0.0.1',
  addHelp: true,
  description: "Imports month-to-date detailed billing reports"
});

parser.addArgument(
  ['--no-stage'], {
    action: 'storeConst',
    dest: 'no_stage',
    help: 'Use an existing staged month-to-date DBR.',
    constant: true
  }
);

var args = parser.parseArgs();

if (args.debug) {
  log.setLevel('debug');
  log.debug("Debugging output enabled.");
} else {
  log.setLevel('info');
}
log.debug(`Resolved invocation arguments were:\n${util.inspect(args)}`);

// Instantiate a DBR object to work with.
var dbrClientOptions = {
  accessKeyId: args.source_key,
  secretAccessKey: args.source_secret
};

var stagingClientOptions = {
  accessKeyId: args.staging_key,
  secretAccessKey: args.staging_secret
};

var dbr = new DBR(dbrClientOptions, stagingClientOptions,
                  args.source_bucket, args.staging_bucket);


// Instantiate a Redshift object to work with.
var redshift = new Redshift(args.redshift_uri, {
      key: args.staging_key,
      secret: args.staging_secret
});

let startTime = moment.utc();

dbr.getMonthToDateDBR()
  .then(stageDBRCheck)
  .then(importDBR)
  .then(vacuum)
  .then(function() {
    cliUtils.runCompleteHandler(startTime, 0);
  })
  .catch(cliUtils.rejectHandler);


// Determine whether to stage the latest month-to-date DBR or reuse existing
function stageDBRCheck(monthToDateDBR) {
  log.info(`Importing ${monthToDateDBR.Date.format("MMMM YYYY")} into month_to_date...`);
  if (args.no_stage) {
    let s3uri = dbr.composeStagedURI(monthToDateDBR);
    log.info(`--no-stage specified, Attempting to use existing staged month-to-date DBR`);
    log.debug(`Importing from ${s3uri}`);
    return s3uri;
  } else {
    return dbr.stageDBR(monthToDateDBR.Date);
  }
}


// Import the staged month-to-date DBR
// TODO if we just chain like .then(redshift.importMonthToDate), it fails
// because "this" inside importMonthToDate will be undefined. Why?
function importDBR(s3uri) {
  return redshift.importMonthToDate(s3uri);
}


// Run VACUUM on the month_to_date table
function vacuum() {
  if (!args.no_vacuum) {
    log.info("Running VACUUM on line_items...");
    return redshift.vacuum('month_to_date');
  } else {
    log.info("--no-vacuum specified, skiping vacuum.");
    return;
  }
}<commit_msg>Herp derp, renamed Date -> Month<commit_after>/* jshint esnext: true, node: true */
'use strict';

/*******************************************************************************
Import month-to-date DBRs, overwriting the existing month-to-date.



*******************************************************************************/



var util = require('util');
var path = require('path');

var log = require('loglevel');
var ArgumentParser = require('argparse').ArgumentParser;
var rollbar = require('rollbar');
var moment = require('moment');

var BaseParser = require('./lib/baseparser.js');
var DBR = require('./lib/dbr.js');
var Redshift = require('./lib/redshift.js');
var cliUtils = require('./lib/cliutils.js');

rollbar.init(process.env.ROLLBAR_TOKEN, {environment: process.env.ROLLBAR_ENVIRONMENT});
rollbar.handleUncaughtExceptions(process.env.ROLLBAR_TOKEN,
                                 {exitOnUncaughtException: true});


var parser = new BaseParser({
  version: '0.0.1',
  addHelp: true,
  description: "Imports month-to-date detailed billing reports"
});

parser.addArgument(
  ['--no-stage'], {
    action: 'storeConst',
    dest: 'no_stage',
    help: 'Use an existing staged month-to-date DBR.',
    constant: true
  }
);

var args = parser.parseArgs();

if (args.debug) {
  log.setLevel('debug');
  log.debug("Debugging output enabled.");
} else {
  log.setLevel('info');
}
log.debug(`Resolved invocation arguments were:\n${util.inspect(args)}`);

// Instantiate a DBR object to work with.
var dbrClientOptions = {
  accessKeyId: args.source_key,
  secretAccessKey: args.source_secret
};

var stagingClientOptions = {
  accessKeyId: args.staging_key,
  secretAccessKey: args.staging_secret
};

var dbr = new DBR(dbrClientOptions, stagingClientOptions,
                  args.source_bucket, args.staging_bucket);


// Instantiate a Redshift object to work with.
var redshift = new Redshift(args.redshift_uri, {
      key: args.staging_key,
      secret: args.staging_secret
});

let startTime = moment.utc();

dbr.getMonthToDateDBR()
  .then(stageDBRCheck)
  .then(importDBR)
  .then(vacuum)
  .then(function() {
    cliUtils.runCompleteHandler(startTime, 0);
  })
  .catch(cliUtils.rejectHandler);


// Determine whether to stage the latest month-to-date DBR or reuse existing
function stageDBRCheck(monthToDateDBR) {
  log.info(`Importing ${monthToDateDBR.Month.format("MMMM YYYY")} into month_to_date...`);
  if (args.no_stage) {
    let s3uri = dbr.composeStagedURI(monthToDateDBR);
    log.info(`--no-stage specified, Attempting to use existing staged month-to-date DBR`);
    log.debug(`Importing from ${s3uri}`);
    return s3uri;
  } else {
    return dbr.stageDBR(monthToDateDBR.Month);
  }
}


// Import the staged month-to-date DBR
// TODO if we just chain like .then(redshift.importMonthToDate), it fails
// because "this" inside importMonthToDate will be undefined. Why?
function importDBR(s3uri) {
  return redshift.importMonthToDate(s3uri);
}


// Run VACUUM on the month_to_date table
function vacuum() {
  if (!args.no_vacuum) {
    log.info("Running VACUUM on line_items...");
    return redshift.vacuum('month_to_date');
  } else {
    log.info("--no-vacuum specified, skiping vacuum.");
    return;
  }
}
{'fraction_non_alphanumeric': 0.18065214031834337, 'fraction_numerical': 0.0027816411682892906, 'mean_word_length': 4.232012934518997, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3358925', 'n_tokens_mistral': 2133, 'n_tokens_neox': 2063, 'n_words': 474}