<commit_before>'use strict';

// ------------------------------------------------------------------------------------------ Dependencies

require('date-utils');
var fs = require("fs");
var _ = require("lodash");
var zlib = require('zlib');
var aws = require('aws-sdk');
var archiver = require('archiver');

// ------------------------------------------------------------------------------------------ Module Exposure

module.exports = function(options) {
	return new S3Storage(options);
}

// ------------------------------------------------------------------------------------------ Module definition

function S3Storage(options) {
	options = options || {};
	this.options = options;

	if(_.isObject(this.options)) {
		this.s3obj = new aws.S3({
			accessKeyId: this.options.S3AccessKeyId,
			secretAccessKey: this.options.S3SecretAccessKey,
			region: this.options.S3Region,
			sslEnabled: this.options.S3SSLEnabled,
			params: {
				Bucket: this.options.S3Bucket
			}
		});
	} else {
		throw "Invalid options provided";
	}
}


S3Storage.prototype.upload = function(file, context, next) {
	try {
		var s3obj = this.s3obj;
		var body = fs.createReadStream(file.path).pipe(zlib.createGzip());
		console.log('s3 upload');
		s3obj.upload({
			Key: file.id,
			Body: body,
			Metadata: {
				json: JSON.stringify(context)
			},
		}, function(err, data) { 
			next(err, context);
		});
	} catch(err) {
		console.log(err);
		next(err, context);
	}
};

S3Storage.prototype.bundle = function(bundle, next) {
	try {
		var s3obj = this.s3obj;
		s3obj.upload({
			Key: bundle.id,
			Body: JSON.stringify(bundle),
			Metadata: {
				json: JSON.stringify(bundle)
			}
		}, function(err, data) { 
			next(err);
		});
	} catch(err) {
		next(err);
	}
};

S3Storage.prototype.archive = function(token, res, next) {<commit_msg>fixes #78 : Add UnitTests for modules Refactoring modules in preparation of unit test creation<commit_after>'use strict';

// ------------------------------------------------------------------------------------------ Dependencies

require('date-utils');
var fs = require("fs");
var _ = require("lodash");
var zlib = require('zlib');
var aws = require('aws-sdk');
var archiver = require('archiver');

// ------------------------------------------------------------------------------------------ Module Exposure

module.exports = function(options) {
	return new S3Storage(options);
}

// ------------------------------------------------------------------------------------------ Module definition

function S3Storage(options) {
	options = options || {};
	this.options = options;

	if(_.isObject(this.options)) {
		this.s3obj = new aws.S3({
			accessKeyId: this.options.S3AccessKeyId,
			secretAccessKey: this.options.S3SecretAccessKey,
			region: this.options.S3Region,
			sslEnabled: this.options.S3SSLEnabled,
			params: {
				Bucket: this.options.S3Bucket
			}
		});
	} else {
		throw "Invalid options provided";
	}
}


S3Storage.prototype.upload = function(file, context, next) {
	try {
		var s3obj = this.s3obj;
		var body = fs.createReadStream(file.path).pipe(zlib.createGzip());
		s3obj.upload({
			Key: file.id,
			Body: body,
			Metadata: {
				json: JSON.stringify(context)
			},
		}, function(err, data) { 
			next(err, context);
		});
	} catch(err) {
		next(err, context);
	}
};

S3Storage.prototype.bundle = function(bundle, next) {
	try {
		var s3obj = this.s3obj;
		s3obj.upload({
			Key: bundle.id,
			Body: JSON.stringify(bundle),
			Metadata: {
				json: JSON.stringify(bundle)
			}
		}, function(err, data) { 
			next(err);
		});
	} catch(err) {
		next(err);
	}
};

S3Storage.prototype.archive = function(token, res, next) {
{'fraction_non_alphanumeric': 0.28117292408879146, 'fraction_numerical': 0.01068785968758564, 'mean_word_length': 2.051839464882943, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6226162', 'n_tokens_mistral': 1239, 'n_tokens_neox': 1117, 'n_words': 222}