<commit_before>UrlCache = require "./UrlCache"
Path = require "path"
fs = require "fs"
async = require "async"
mkdirp = require "mkdirp"
OutputFileFinder = require "./OutputFileFinder"
Metrics = require "./Metrics"

module.exports = ResourceWriter =
	syncResourcesToDisk: (project_id, resources, basePath, callback = (error) ->) ->
		@_removeExtraneousFiles resources, basePath, (error) =>
			return callback(error) if error?
			jobs = for resource in resources
				do (resource) =>
					(callback) => @_writeResourceToDisk(project_id, resource, basePath, callback)
			async.series jobs, callback

	_removeExtraneousFiles: (resources, basePath, _callback = (error) ->) ->
		timer = new Metrics.Timer("unlink-output-files")
		callback = (error) ->
			timer.done()
			_callback(error)

		OutputFileFinder.findOutputFiles resources, basePath, (error, outputFiles) ->
			return callback(error) if error?

			jobs = []
			for file in outputFiles or []
				do (file) ->
					path = file.path
					should_delete = true
					if path.match(/^output\./) or path.match(/\.aux$/)
						should_delete = false
					if path == "output.pdf" or path == "output.dvi" or path == "output.log"
						should_delete = true
					if should_delete
						jobs.push (callback) -> ResourceWriter._deleteFileIfNotDirectory Path.join(basePath, path), callback

			async.series jobs, callback

	_deleteFileIfNotDirectory: (path, callback = (error) ->) ->
		fs.stat path, (error, stat) ->
			return callback(error) if error?
			if stat.isFile()
				fs.unlink path, callback
			else
				callback()

	_writeResourceToDisk: (project_id, resource, basePath, callback = (error) ->) ->
		path = Path.normalize(Path.join(basePath, resource.path))
		if (path.slice(0, basePath.length) != basePath)
			return callback new Error("resource path is outside root directory")

		mkdirp Path.dirname(path), (error) ->
			return callback(error) if error?
			# TODO: Don't overwrite file if it hasn't been modified
			if resource.url?
				UrlCache.downloadUrlToFile(
					project_id,
					resource.url,
					path,
					resource.modified,
					callback
				)
			else
				fs.writeFile path, resource.content, callback

		
<commit_msg>don't error if a http resource can not be download<commit_after>UrlCache = require "./UrlCache"
Path = require "path"
fs = require "fs"
async = require "async"
mkdirp = require "mkdirp"
OutputFileFinder = require "./OutputFileFinder"
Metrics = require "./Metrics"
logger = require "logger-sharelatex"

module.exports = ResourceWriter =
	syncResourcesToDisk: (project_id, resources, basePath, callback = (error) ->) ->
		@_removeExtraneousFiles resources, basePath, (error) =>
			return callback(error) if error?
			jobs = for resource in resources
				do (resource) =>
					(callback) => @_writeResourceToDisk(project_id, resource, basePath, callback)
			async.series jobs, callback

	_removeExtraneousFiles: (resources, basePath, _callback = (error) ->) ->
		timer = new Metrics.Timer("unlink-output-files")
		callback = (error) ->
			timer.done()
			_callback(error)

		OutputFileFinder.findOutputFiles resources, basePath, (error, outputFiles) ->
			return callback(error) if error?

			jobs = []
			for file in outputFiles or []
				do (file) ->
					path = file.path
					should_delete = true
					if path.match(/^output\./) or path.match(/\.aux$/)
						should_delete = false
					if path == "output.pdf" or path == "output.dvi" or path == "output.log"
						should_delete = true
					if should_delete
						jobs.push (callback) -> ResourceWriter._deleteFileIfNotDirectory Path.join(basePath, path), callback

			async.series jobs, callback

	_deleteFileIfNotDirectory: (path, callback = (error) ->) ->
		fs.stat path, (error, stat) ->
			return callback(error) if error?
			if stat.isFile()
				fs.unlink path, callback
			else
				callback()

	_writeResourceToDisk: (project_id, resource, basePath, callback = (error) ->) ->
		path = Path.normalize(Path.join(basePath, resource.path))
		if (path.slice(0, basePath.length) != basePath)
			return callback new Error("resource path is outside root directory")

		mkdirp Path.dirname(path), (error) ->
			return callback(error) if error?
			# TODO: Don't overwrite file if it hasn't been modified
			if resource.url?
				UrlCache.downloadUrlToFile project_id, resource.url, path, resource.modified, (err)->
					if err?
						logger.err err:err, "error downloading file for resources"
					callback() #try and continue compiling even if http resource can not be downloaded at this time
			else
				fs.writeFile path, resource.content, callback

		

{'fraction_non_alphanumeric': 0.11910994764397906, 'fraction_numerical': 0.0004363001745200698, 'mean_word_length': 3.6977459016393444, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1361655', 'n_tokens_mistral': 1633, 'n_tokens_neox': 1362, 'n_words': 403}