<commit_before>var _ = require("underscore");
var knox = require("knox");
var fs = require("fs");

module.exports = {

	runningJobs: 0,

	client: null,

	s3Config: function(supplied_config){
		var error = "Cannot find s3 settings in config";

		if(_.has(supplied_config, "publish") && _.has(supplied_config["publish"], "s3"))
			return supplied_config["publish"]["s3"]
		else 
			throw error;
	},

	forEachFileIn: function(dir, callback){

		var self = this;

		fs.readdir(dir, function(read_err, files){
			if(read_err)	
				throw read_err;


			_.each(files, function(file){
				var file_path = dir + "/" + file;

				// check for sub directories
				fs.stat(file_path, function(stat_err, stats){

					if(stat_err)
						throw stat_err;

					if(stats.isDirectory()){
						// iterate inside the directory	
						self.forEachFileIn(file_path, callback);
					} else {
						// execute the callback	
						callback(file_path);
					}

				});	
			});
		});

	},

	copyFile: function(file_path){

		var self = this;

		fs.readFile(file_path, function(error, buf){

			if(error)
				throw error;

			// increment running jobs
			self.runningJobs++;

			var file_path_in_array = file_path.split("/");
			file_path_in_array.shift();
			file_path_without_output_dir = file_path_in_array.join("/");

			var req = self.client.put(file_path_without_output_dir, {
				'Content-Length': buf.length
			, 'Content-Type': 'text/plain'
			});

			req.on('response', function(res){
				if (200 == res.statusCode) {
					console.log('saved to %s', req.url);
				} else {
					console.log('error occured in copying to %s', req.url);	
				}

				// decrement running jobs
				self.runningJobs--;

			});

			req.end(buf);
		});
	},

	publish: function(supplied_config){

		var self = this;

		// create client
		self.client = knox.createClient(self.s3Config(supplied_config));

		// reset the running job count
		self.runningJobs = 0;
		
		var output_dir = supplied_config.output_dir

		self.forEachFileIn(output_dir, function(file_path){
			// copy the file to the bucket
			self.copyFile(file_path);
		});

	}
}


<commit_msg>set the correct mime-type to the files published<commit_after>var _ = require("underscore");
var knox = require("knox");
var mime = require("mime");
var fs = require("fs");

module.exports = {

	runningJobs: 0,

	client: null,

	s3Config: function(supplied_config){
		var error = "Cannot find s3 settings in config";

		if(_.has(supplied_config, "publish") && _.has(supplied_config["publish"], "s3"))
			return supplied_config["publish"]["s3"]
		else 
			throw error;
	},

	forEachFileIn: function(dir, callback){

		var self = this;

		fs.readdir(dir, function(read_err, files){
			if(read_err)	
				throw read_err;


			_.each(files, function(file){
				var file_path = dir + "/" + file;

				// check for sub directories
				fs.stat(file_path, function(stat_err, stats){

					if(stat_err)
						throw stat_err;

					if(stats.isDirectory()){
						// iterate inside the directory	
						self.forEachFileIn(file_path, callback);
					} else {
						// execute the callback	
						callback(file_path);
					}

				});	
			});
		});

	},

	copyFile: function(file_path){

		var self = this;

		fs.readFile(file_path, function(error, buf){

			if(error)
				throw error;

			// increment running jobs
			self.runningJobs++;

			var file_path_in_array = file_path.split("/");
			file_path_in_array.shift();
			file_path_without_output_dir = file_path_in_array.join("/");

			var req = self.client.put(file_path_without_output_dir, {
				'Content-Length': buf.length
			, 'Content-Type': mime.lookup(file_path)
			});

			req.on('response', function(res){
				if (200 == res.statusCode) {
					console.log('saved to %s', req.url);
				} else {
					console.log('error occured in copying to %s', req.url);	
				}

				// decrement running jobs
				self.runningJobs--;

			});

			req.end(buf);
		});
	},

	publish: function(supplied_config){

		var self = this;

		// create client
		self.client = knox.createClient(self.s3Config(supplied_config));

		// reset the running job count
		self.runningJobs = 0;
		
		var output_dir = supplied_config.output_dir

		self.forEachFileIn(output_dir, function(file_path){
			// copy the file to the bucket
			self.copyFile(file_path);
		});

	}
}



{'fraction_non_alphanumeric': 0.1407959041191529, 'fraction_numerical': 0.004654410053525715, 'mean_word_length': 3.17687074829932, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4585038', 'n_tokens_mistral': 1805, 'n_tokens_neox': 1442, 'n_words': 312}