<commit_before>async = require "async"
fs = require "fs"
Path = require "path"
spawn = require("child_process").spawn
logger = require "logger-sharelatex"

module.exports = OutputFileFinder =
	findOutputFiles: (resources, directory, callback = (error, outputFiles) ->) ->
		incomingResources = {}
		for resource in resources
			incomingResources[resource.path] = true
			
		logger.log directory: directory, "getting output files"

		OutputFileFinder._getAllFiles directory, (error, allFiles = []) ->
			return callback(error) if error?
			jobs = []
			outputFiles = []
			for file in allFiles
				do (file) ->
					jobs.push (callback) ->
						if incomingResources[file]
							return callback()
						else
							outputFiles.push {
								path: file
								type: file.match(/\.([^\.]+)$/)?[1]
							}
							callback()

			async.series jobs, (error) ->
				return callback(error) if error?
				callback null, outputFiles

	_getAllFiles: (directory, _callback = (error, fileList) ->) ->
		callback = (error, fileList) ->
			_callback(error, fileList)
			_callback = () ->
				
		args = [directory, "-type", "f"]
		logger.log args: args, "running find command"

		proc = spawn("find", args)
		stdout = ""
		proc.stdout.on "data", (chunk) ->
			stdout += chunk.toString()	
		proc.on "error", callback	
		proc.on "close", (code) ->
			if code != 0
				logger.warn {directory, code}, "find returned error, directory likely doesn't exist"
				return callback null, []
			fileList = stdout.trim().split("\n")
			fileList = fileList.map (file) ->
				# Strip leading directory
				path = Path.relative(directory, file)
			return callback null, fileList

<commit_msg>skip the cache directory when finding output files<commit_after>async = require "async"
fs = require "fs"
Path = require "path"
spawn = require("child_process").spawn
logger = require "logger-sharelatex"

module.exports = OutputFileFinder =
	findOutputFiles: (resources, directory, callback = (error, outputFiles) ->) ->
		incomingResources = {}
		for resource in resources
			incomingResources[resource.path] = true
			
		logger.log directory: directory, "getting output files"

		OutputFileFinder._getAllFiles directory, (error, allFiles = []) ->
			return callback(error) if error?
			jobs = []
			outputFiles = []
			for file in allFiles
				do (file) ->
					jobs.push (callback) ->
						if incomingResources[file]
							return callback()
						else
							outputFiles.push {
								path: file
								type: file.match(/\.([^\.]+)$/)?[1]
							}
							callback()

			async.series jobs, (error) ->
				return callback(error) if error?
				callback null, outputFiles

	_getAllFiles: (directory, _callback = (error, fileList) ->) ->
		callback = (error, fileList) ->
			_callback(error, fileList)
			_callback = () ->
				
		args = [directory, "-name", ".cache", "-prune", "-o", "-type", "f", "-print"]
		logger.log args: args, "running find command"

		proc = spawn("find", args)
		stdout = ""
		proc.stdout.on "data", (chunk) ->
			stdout += chunk.toString()	
		proc.on "error", callback	
		proc.on "close", (code) ->
			if code != 0
				logger.warn {directory, code}, "find returned error, directory likely doesn't exist"
				return callback null, []
			fileList = stdout.trim().split("\n")
			fileList = fileList.map (file) ->
				# Strip leading directory
				path = Path.relative(directory, file)
			return callback null, fileList


{'fraction_non_alphanumeric': 0.1414319248826291, 'fraction_numerical': 0.0011737089201877935, 'mean_word_length': 3.1573170731707316, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4619756', 'n_tokens_mistral': 1274, 'n_tokens_neox': 1020, 'n_words': 294}