<commit_before>async = require "async"
fs = require "fs"
fse = require "fs-extra"
Path = require "path"
logger = require "logger-sharelatex"
_ = require "underscore"
Settings = require "settings-sharelatex"
crypto = require "crypto"

OutputFileOptimiser = require "./OutputFileOptimiser"

module.exports = OutputCacheManager =
	CACHE_SUBDIR: '.cache/clsi'
	ARCHIVE_SUBDIR: '.archive/clsi'
	# build id is HEXDATE-HEXRANDOM from Date.now()and RandomBytes
	# for backwards compatibility, make the randombytes part optional
	BUILD_REGEX: /^[0-9a-f]+(-[0-9a-f]+)?$/
	CACHE_LIMIT: 2  # maximum number of cache directories
	CACHE_AGE: 60*60*1000 # up to one hour old

	path: (buildId, file) ->
		# used by static server, given build id return '.cache/clsi/buildId'
		if buildId.match OutputCacheManager.BUILD_REGEX
			return Path.join(OutputCacheManager.CACHE_SUBDIR, buildId, file)
		else
			# for invalid build id, return top level
			return file

	generateBuildId: (callback = (error, buildId) ->) ->
		# generate a secure build id from Date.now() and 8 random bytes in hex
		crypto.randomBytes 8, (err, buf) ->
			return callback(err) if err?
			random = buf.toString('hex')
			date = Date.now().toString(16)
			callback err, "#{date}-#{random}"

	saveOutputFiles: (outputFiles, compileDir, callback = (error) ->) ->
		OutputCacheManager.generateBuildId (err, buildId) ->
			return callback(err) if err?
			OutputCacheManager.saveOutputFilesInBuildDir outputFiles, compileDir, buildId, callback

	saveOutputFilesInBuildDir: (outputFiles, compileDir, buildId, callback = (error) ->) ->
		# make a compileDir/CACHE_SUBDIR/build_id directory and
		# copy all the output files into it
		cacheRoot = Path.join(compileDir, OutputCacheManager.CACHE_SUBDIR)
		# Put the files into a new cache subdirectory
		cacheDir = Path.join(compileDir, OutputCacheManager.CACHE_SUBDIR, buildId)
		# Is it a per-user compile? check if compile directory is PROJECTID-USERID
		perUser = Path.basename(compileDir).match(/^[0-9a-f]{24}-[0-9a-f]{24}$/)

		# Archive logs in background
		if Settings.clsi?.archive_logs or Settings.clsi?.strace
			OutputCacheManager.archiveLogs outputFiles, compileDir, buildId, (err) ->
				if err?
					logger.warn err:err, "erroring archiving log files"

		# make the new cache directory
		fse.ensureDir cacheDir, (err) ->
			if err?
				logger.error err: err, directory: cacheDir, "error creating cache directory"
				callback(err, outputFiles)
			else
				# copy all the output files into the new cache directory
				results = []
				async.mapSeries outputFiles, (file, cb) ->
					newFile = _.clone(file)
					[src, dst] = [Path.join(compileDir, file.path), Path.join(cacheDir, file.path)]
					OutputCacheManager._checkFileIsSafe src, (err, isSafe) ->
						return cb(err) if err?
						if !isSafe
							return cb()
						OutputCacheManager._checkIfShouldCopy src, (err, shouldCopy) ->
							return cb(err) if err?
							if !shouldCopy
								return cb()
							OutputCacheManager._copyFile src, dst, (err) ->
								return cb(err) if err?
								newFile.build = buildId  # attach a build id if we cached the file
								results.push newFile
								cb()
				, (err) ->
					if err?
						# pass back the original files if we encountered *any* error
						callback(err, outputFiles)
						# clean up the directory we just created
						fse.remove cacheDir, (err) ->
							if err?
								logger.error err: err, dir: dir, "error removing cache dir after failure"
					else
						# pass back the list of new files in the cache
						callback(err, results)
						# let file expiry run in the background, expire all previous files if per-user
						OutputCacheManager.expireOutputFiles cacheRoot, {keep: buildId, limit: if perUser then 0 else null}

	archiveLogs: (outputFiles, compileDir, buildId, callback = (error) ->) ->
		archiveDir = Path.join(compileDir, OutputCacheManager.ARCHIVE_SUBDIR, buildId)
		logger.log {dir: archiveDir}, "archiving log files for project"
		fse.ensureDir archiveDir, (err) ->
			return callback(err) if err?
			async.mapSeries outputFiles, (file, cb) ->
				[src, dst] = [Path.join(compileDir, file.path), Path.join(archiveDir, file.path)]
				OutputCacheManager._checkFileIsSafe src, (err, isSafe) ->
					return cb(err) if err?
					return cb() if !isSafe
					OutputCacheManager._checkIfShouldArchive src, (err, shouldArchive) ->
						return cb(err) if err?
						return cb() if !shouldArchive
						OutputCacheManager._copyFile src, dst, cb
			, callback

	expireOutputFiles: (cacheRoot, options, callback = (error) ->) ->
		# look in compileDir for build dirs and delete if > N or age of mod time > T
		fs.readdir cacheRoot, (err, results) ->
			if err?
				return callback(null) if err.code == 'ENOENT'	# cache directory is empty
				logger.error err: err, project_id: cacheRoot, "error clearing cache"
				return callback(err)

			dirs =  results.sort().reverse()
			currentTime = Date.now()

			isExpired = (dir, index) ->
				return false if options?.keep == dir<commit_msg>keep one extra build until per-page pdf serving is enabled<commit_after>async = require "async"
fs = require "fs"
fse = require "fs-extra"
Path = require "path"
logger = require "logger-sharelatex"
_ = require "underscore"
Settings = require "settings-sharelatex"
crypto = require "crypto"

OutputFileOptimiser = require "./OutputFileOptimiser"

module.exports = OutputCacheManager =
	CACHE_SUBDIR: '.cache/clsi'
	ARCHIVE_SUBDIR: '.archive/clsi'
	# build id is HEXDATE-HEXRANDOM from Date.now()and RandomBytes
	# for backwards compatibility, make the randombytes part optional
	BUILD_REGEX: /^[0-9a-f]+(-[0-9a-f]+)?$/
	CACHE_LIMIT: 2  # maximum number of cache directories
	CACHE_AGE: 60*60*1000 # up to one hour old

	path: (buildId, file) ->
		# used by static server, given build id return '.cache/clsi/buildId'
		if buildId.match OutputCacheManager.BUILD_REGEX
			return Path.join(OutputCacheManager.CACHE_SUBDIR, buildId, file)
		else
			# for invalid build id, return top level
			return file

	generateBuildId: (callback = (error, buildId) ->) ->
		# generate a secure build id from Date.now() and 8 random bytes in hex
		crypto.randomBytes 8, (err, buf) ->
			return callback(err) if err?
			random = buf.toString('hex')
			date = Date.now().toString(16)
			callback err, "#{date}-#{random}"

	saveOutputFiles: (outputFiles, compileDir, callback = (error) ->) ->
		OutputCacheManager.generateBuildId (err, buildId) ->
			return callback(err) if err?
			OutputCacheManager.saveOutputFilesInBuildDir outputFiles, compileDir, buildId, callback

	saveOutputFilesInBuildDir: (outputFiles, compileDir, buildId, callback = (error) ->) ->
		# make a compileDir/CACHE_SUBDIR/build_id directory and
		# copy all the output files into it
		cacheRoot = Path.join(compileDir, OutputCacheManager.CACHE_SUBDIR)
		# Put the files into a new cache subdirectory
		cacheDir = Path.join(compileDir, OutputCacheManager.CACHE_SUBDIR, buildId)
		# Is it a per-user compile? check if compile directory is PROJECTID-USERID
		perUser = Path.basename(compileDir).match(/^[0-9a-f]{24}-[0-9a-f]{24}$/)

		# Archive logs in background
		if Settings.clsi?.archive_logs or Settings.clsi?.strace
			OutputCacheManager.archiveLogs outputFiles, compileDir, buildId, (err) ->
				if err?
					logger.warn err:err, "erroring archiving log files"

		# make the new cache directory
		fse.ensureDir cacheDir, (err) ->
			if err?
				logger.error err: err, directory: cacheDir, "error creating cache directory"
				callback(err, outputFiles)
			else
				# copy all the output files into the new cache directory
				results = []
				async.mapSeries outputFiles, (file, cb) ->
					newFile = _.clone(file)
					[src, dst] = [Path.join(compileDir, file.path), Path.join(cacheDir, file.path)]
					OutputCacheManager._checkFileIsSafe src, (err, isSafe) ->
						return cb(err) if err?
						if !isSafe
							return cb()
						OutputCacheManager._checkIfShouldCopy src, (err, shouldCopy) ->
							return cb(err) if err?
							if !shouldCopy
								return cb()
							OutputCacheManager._copyFile src, dst, (err) ->
								return cb(err) if err?
								newFile.build = buildId  # attach a build id if we cached the file
								results.push newFile
								cb()
				, (err) ->
					if err?
						# pass back the original files if we encountered *any* error
						callback(err, outputFiles)
						# clean up the directory we just created
						fse.remove cacheDir, (err) ->
							if err?
								logger.error err: err, dir: dir, "error removing cache dir after failure"
					else
						# pass back the list of new files in the cache
						callback(err, results)
						# let file expiry run in the background, expire all previous files if per-user
						OutputCacheManager.expireOutputFiles cacheRoot, {keep: buildId, limit: if perUser then 1 else null}

	archiveLogs: (outputFiles, compileDir, buildId, callback = (error) ->) ->
		archiveDir = Path.join(compileDir, OutputCacheManager.ARCHIVE_SUBDIR, buildId)
		logger.log {dir: archiveDir}, "archiving log files for project"
		fse.ensureDir archiveDir, (err) ->
			return callback(err) if err?
			async.mapSeries outputFiles, (file, cb) ->
				[src, dst] = [Path.join(compileDir, file.path), Path.join(archiveDir, file.path)]
				OutputCacheManager._checkFileIsSafe src, (err, isSafe) ->
					return cb(err) if err?
					return cb() if !isSafe
					OutputCacheManager._checkIfShouldArchive src, (err, shouldArchive) ->
						return cb(err) if err?
						return cb() if !shouldArchive
						OutputCacheManager._copyFile src, dst, cb
			, callback

	expireOutputFiles: (cacheRoot, options, callback = (error) ->) ->
		# look in compileDir for build dirs and delete if > N or age of mod time > T
		fs.readdir cacheRoot, (err, results) ->
			if err?
				return callback(null) if err.code == 'ENOENT'	# cache directory is empty
				logger.error err: err, project_id: cacheRoot, "error clearing cache"
				return callback(err)

			dirs =  results.sort().reverse()
			currentTime = Date.now()

			isExpired = (dir, index) ->
				return false if options?.keep == dir
{'fraction_non_alphanumeric': 0.10695557534382111, 'fraction_numerical': 0.005144949045216187, 'mean_word_length': 3.5966348340154615, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '803881', 'n_tokens_mistral': 3688, 'n_tokens_neox': 3083, 'n_words': 1000}