<commit_before>settings = require "settings-sharelatex"
logger = require "logger-sharelatex"
AWS = require 'aws-sdk'
S3S = require 's3-streams'
{db, ObjectId} = require "./mongojs"
JSONStream = require "JSONStream"
ReadlineStream = require "readline-stream"

module.exports = MongoAWS =

	MAX_SIZE:  1024*1024 # almost max size
	MAX_COUNT: 1024 # almost max count

	archiveDocHistory: (project_id, doc_id, update, _callback = (error) ->) ->

		callback = (args...) ->
			_callback(args...)
			_callback = () ->

		query = {
			doc_id: ObjectId(doc_id)
			v: {$lt: update.v}
			expiresAt: {$exists : false}
		}

		AWS.config.update {
			accessKeyId: settings.filestore.s3.key
			secretAccessKey: settings.filestore.s3.secret
		}
		 
		upload = S3S.WriteStream new AWS.S3(), {
			"Bucket": settings.filestore.stores.user_files,
			"Key": project_id+"/changes-"+doc_id
		}

		db.docHistory.find(query)
			.on 'error', (err) ->
				callback(err)
			.pipe JSONStream.stringify()
			.pipe upload
			.on 'error', (err) ->
				callback(err)<commit_msg>respect mongo bulk operations limit of 1000 operations<commit_after>settings = require "settings-sharelatex"
logger = require "logger-sharelatex"
AWS = require 'aws-sdk'
S3S = require 's3-streams'
{db, ObjectId} = require "./mongojs"
JSONStream = require "JSONStream"
ReadlineStream = require "readline-stream"

module.exports = MongoAWS =

	MAX_SIZE:  1024*1024 # almost max size
	MAX_COUNT: 512 # almost max count

	archiveDocHistory: (project_id, doc_id, update, _callback = (error) ->) ->

		callback = (args...) ->
			_callback(args...)
			_callback = () ->

		query = {
			doc_id: ObjectId(doc_id)
			v: {$lt: update.v}
			expiresAt: {$exists : false}
		}

		AWS.config.update {
			accessKeyId: settings.filestore.s3.key
			secretAccessKey: settings.filestore.s3.secret
		}
		 
		upload = S3S.WriteStream new AWS.S3(), {
			"Bucket": settings.filestore.stores.user_files,
			"Key": project_id+"/changes-"+doc_id
		}

		db.docHistory.find(query)
			.on 'error', (err) ->
				callback(err)
			.pipe JSONStream.stringify()
			.pipe upload
			.on 'error', (err) ->
				callback(err)
{'fraction_non_alphanumeric': 0.14567669172932332, 'fraction_numerical': 0.01832706766917293, 'mean_word_length': 3.5686695278969958, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2729339', 'n_tokens_mistral': 874, 'n_tokens_neox': 749, 'n_words': 164}