<commit_before>settings = require("settings-sharelatex")
request = require("request")
logger = require("logger-sharelatex")
fs = require("fs")
knox = require("knox")
path = require("path")
LocalFileWriter = require("./LocalFileWriter")
_ = require("underscore")
	

thirtySeconds = 30 * 1000

buildDefaultOptions = (bucketName, method, key)->
	return {
			aws:
				key: settings.s3.key
				secret: settings.s3.secret
				bucket: bucketName
			method: method
			timeout: thirtySeconds
			uri:"https://#{bucketName}.s3.amazonaws.com/#{key}"
	}

module.exports =

	sendFileToS3: (bucketName, key, fsPath, callback)->
		s3Client = knox.createClient
			key: settings.s3.key
			secret: settings.s3.secret
			bucket: bucketName
		putEventEmiter = s3Client.putFile fsPath, key, (err, res)->
			if err?
				logger.err err:err,  bucketName:bucketName, key:key, fsPath:fsPath,"something went wrong uploading file to s3"
				return callback(err)
			if !res?
				logger.err err:err, res:res, bucketName:bucketName, key:key, fsPath:fsPath, "no response from s3 put file"
				callback("no response from put file")
			if res.statusCode != 200
				logger.err bucketName:bucketName, key:key, fsPath:fsPath, "non 200 response from s3 putting file"
				return callback("non 200 response from s3 on put file")
			LocalFileWriter.deleteFile fsPath, (err)->
				logger.log res:res,  bucketName:bucketName, key:key, fsPath:fsPath,"file uploaded to s3"
				callback(err)
		putEventEmiter.on "error", (err)->
			logger.err err:err,  bucketName:bucketName, key:key, fsPath:fsPath, "error emmited on put of file"
			callback err


	sendStreamToS3: (bucketName, key, readStream, callback)->
		logger.log bucketName:bucketName, key:key, "sending file to s3"
		readStream.on "error", (err)->
			logger.err bucketName:bucketName, key:key, "error on stream to send to s3"
		LocalFileWriter.writeStream readStream, null, (err, fsPath)=>
			if err?
				logger.err  bucketName:bucketName, key:key, fsPath:fsPath, err:err, "something went wrong writing stream to disk"
				return callback(err)
			@sendFileToS3 bucketName, key, fsPath, callback
			
	getFileStream: (bucketName, key, callback)->
		logger.log bucketName:bucketName, key:key, "getting file from s3"
		options = buildDefaultOptions(bucketName, "get", key)
		readStream = request(options)
		readStream.on "error", (err)->
			logger.err err:err, bucketName:bucketName, key:key, "error getting file stream from s3"
		callback null, readStream

	copyFile: (bucketName, sourceKey, destKey, callback)->
		logger.log bucketName:bucketName, sourceKey:sourceKey, destKey:destKey, "copying file in s3"
		s3Client = knox.createClient
			key: settings.s3.key
			secret: settings.s3.secret
			bucket: bucketName
		s3Client.copyFile sourceKey, destKey, (err)->
			if err?
				logger.err err:err, bucketName:bucketName, sourceKey:sourceKey, destKey:destKey, "something went wrong copying file in aws"
			callback(err)

	deleteFile: (bucketName, key, callback)->
		logger.log bucketName:bucketName, key:key, "delete file in s3"
		options = buildDefaultOptions(bucketName, "delete", key)
		request options, (err, res)->
			if err?
				logger.err err:err, res:res, bucketName:bucketName, key:key, "something went wrong deleting file in aws"
			callback(err)

	deleteDirectory: (bucketName, key, callback)->
		s3Client = knox.createClient
			key: settings.s3.key
			secret: settings.s3.secret
			bucket: bucketName
		s3Client.list prefix:key, (err, data)->
			keys = _.map data.Contents, (entry)->
				return entry.Key
			s3Client.deleteMultiple keys, callback

	checkIfFileExists:(bucketName, key, callback)->
		logger.log bucketName:bucketName, key:key, "checking if file exists in s3"
		options = buildDefaultOptions(bucketName, "head", key)
		request options, (err, res)->
			if err?
				logger.err err:err, res:res, bucketName:bucketName, key:key, "something went wrong copying file in aws"
			if !res?
				logger.err err:err, res:res, bucketName:bucketName, key:key, "no response object returned when checking if file exists"
			exists = res.statusCode == 200
			logger.log bucketName:bucketName, key:key, exists:exists, "checked if file exsists in s3"<commit_msg>call the callback when no response is returned from s3 when checking if file exists<commit_after>settings = require("settings-sharelatex")
request = require("request")
logger = require("logger-sharelatex")
fs = require("fs")
knox = require("knox")
path = require("path")
LocalFileWriter = require("./LocalFileWriter")
_ = require("underscore")
	

thirtySeconds = 30 * 1000

buildDefaultOptions = (bucketName, method, key)->
	return {
			aws:
				key: settings.s3.key
				secret: settings.s3.secret
				bucket: bucketName
			method: method
			timeout: thirtySeconds
			uri:"https://#{bucketName}.s3.amazonaws.com/#{key}"
	}

module.exports =

	sendFileToS3: (bucketName, key, fsPath, callback)->
		s3Client = knox.createClient
			key: settings.s3.key
			secret: settings.s3.secret
			bucket: bucketName
		putEventEmiter = s3Client.putFile fsPath, key, (err, res)->
			if err?
				logger.err err:err,  bucketName:bucketName, key:key, fsPath:fsPath,"something went wrong uploading file to s3"
				return callback(err)
			if !res?
				logger.err err:err, res:res, bucketName:bucketName, key:key, fsPath:fsPath, "no response from s3 put file"
				callback("no response from put file")
			if res.statusCode != 200
				logger.err bucketName:bucketName, key:key, fsPath:fsPath, "non 200 response from s3 putting file"
				return callback("non 200 response from s3 on put file")
			LocalFileWriter.deleteFile fsPath, (err)->
				logger.log res:res,  bucketName:bucketName, key:key, fsPath:fsPath,"file uploaded to s3"
				callback(err)
		putEventEmiter.on "error", (err)->
			logger.err err:err,  bucketName:bucketName, key:key, fsPath:fsPath, "error emmited on put of file"
			callback err


	sendStreamToS3: (bucketName, key, readStream, callback)->
		logger.log bucketName:bucketName, key:key, "sending file to s3"
		readStream.on "error", (err)->
			logger.err bucketName:bucketName, key:key, "error on stream to send to s3"
		LocalFileWriter.writeStream readStream, null, (err, fsPath)=>
			if err?
				logger.err  bucketName:bucketName, key:key, fsPath:fsPath, err:err, "something went wrong writing stream to disk"
				return callback(err)
			@sendFileToS3 bucketName, key, fsPath, callback
			
	getFileStream: (bucketName, key, callback)->
		logger.log bucketName:bucketName, key:key, "getting file from s3"
		options = buildDefaultOptions(bucketName, "get", key)
		readStream = request(options)
		readStream.on "error", (err)->
			logger.err err:err, bucketName:bucketName, key:key, "error getting file stream from s3"
		callback null, readStream

	copyFile: (bucketName, sourceKey, destKey, callback)->
		logger.log bucketName:bucketName, sourceKey:sourceKey, destKey:destKey, "copying file in s3"
		s3Client = knox.createClient
			key: settings.s3.key
			secret: settings.s3.secret
			bucket: bucketName
		s3Client.copyFile sourceKey, destKey, (err)->
			if err?
				logger.err err:err, bucketName:bucketName, sourceKey:sourceKey, destKey:destKey, "something went wrong copying file in aws"
			callback(err)

	deleteFile: (bucketName, key, callback)->
		logger.log bucketName:bucketName, key:key, "delete file in s3"
		options = buildDefaultOptions(bucketName, "delete", key)
		request options, (err, res)->
			if err?
				logger.err err:err, res:res, bucketName:bucketName, key:key, "something went wrong deleting file in aws"
			callback(err)

	deleteDirectory: (bucketName, key, callback)->
		s3Client = knox.createClient
			key: settings.s3.key
			secret: settings.s3.secret
			bucket: bucketName
		s3Client.list prefix:key, (err, data)->
			keys = _.map data.Contents, (entry)->
				return entry.Key
			s3Client.deleteMultiple keys, callback

	checkIfFileExists:(bucketName, key, callback)->
		logger.log bucketName:bucketName, key:key, "checking if file exists in s3"
		options = buildDefaultOptions(bucketName, "head", key)
		request options, (err, res)->
			if err?
				logger.err err:err, res:res, bucketName:bucketName, key:key, "something went wrong copying file in aws"
			if !res?
				logger.err err:err, res:res, bucketName:bucketName, key:key, "no response object returned when checking if file exists"
				err = new Error("no response from s3 #{bucketName} #{key}")
				return callback(err)
			exists = res.statusCode == 200
			logger.log bucketName:bucketName, key:key, exists:exists, "checked if file exsists in s3"
{'fraction_non_alphanumeric': 0.11549759135236752, 'fraction_numerical': 0.011984490659146986, 'mean_word_length': 4.652058432934927, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 41, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2642643', 'n_tokens_mistral': 3048, 'n_tokens_neox': 2694, 'n_words': 795}