<commit_before>import org.codehaus.groovy.grails.commons.ApplicationHolder
import org.apache.tools.ant.DirectoryScanner
// import asset.pipeline.*;

includeTargets << grailsScript("_GrailsBootstrap")

target(assetPrecompile: "Precompiles assets in the application as specified by the precompile glob!") {
    depends(configureProxy,compile, packageApp, bootstrap)

    def assetHelper             = classLoader.loadClass('asset.pipeline.AssetHelper')
    def directiveProcessorClass = classLoader.loadClass('asset.pipeline.DirectiveProcessor')
    def uglifyJsProcessor 		= classLoader.loadClass('asset.pipeline.processors.UglifyJsProcessor').newInstance()
	def grailsApplication       = ApplicationHolder.getApplication()
    event("StatusUpdate",["Precompiling Assets!"]);

	// TODO: Find all Files we want to process
	DirectoryScanner scanner = new DirectoryScanner()
	scanner.setExcludes(["**/.*","**/.DS_Store"] as String[])
	scanner.setIncludes(["**/*"] as String[])

	def assetPaths = assetHelper.getAssetPaths()
	def filesToProcess = []
	for(path in assetPaths) {
	    event("StatusUpdate",["Scanning ${path}"]);
	    scanner.setBasedir(path)
	    scanner.setCaseSensitive(false)
	    scanner.scan()
	    filesToProcess += scanner.getIncludedFiles().flatten()

	}
	filesToProcess.unique()

	for(counter = 0 ; counter < filesToProcess.size(); counter++) {
		def fileName = filesToProcess[counter]
		event("StatusUpdate",["Processing File ${counter+1} of ${filesToProcess.size()} - ${fileName}"]);
		def extension = assetHelper.extensionFromURI(fileName)
		fileName = fileName.substring(0,fileName.lastIndexOf("."))

		def assetFile = assetHelper.artefactForFileWithExtension(assetHelper.fileForUri(fileName,null,extension), extension)
		if(assetFile) {
			def fileData = null

			if(assetFile.class.name == 'java.io.File') {
				// println "Non Asset File"
			}
			else {
				def directiveProcessor = directiveProcessorClass.newInstance(assetFile.contentType)
				fileData = directiveProcessor.compile(assetFile)
				if(assetFile.contentType == 'application/javascript') {
					def newFileData = fileData
					try {
						newFileData = uglifyJsProcessor.process(fileData)
					} catch(e) {
						println "Uglify JS Exception ${e}"
						newFileData = fileData
					}
					fileData = newFileData
				}
			}


			def outputFile = new File("web-app/assets/${fileName}.${extension}");
			def parentTree = new File(outputFile.parent)
			parentTree.mkdirs()
			outputFile.createNewFile();
			if(fileData) {
				outputFile.text = fileData
			} else {
				if(assetFile.class.name == 'java.io.File') {
					outputFile.bytes = assetFile.bytes
				} else {
					outputFile.bytes = assetFile.file.bytes
				}
			}
			def targetStream = new java.io.ByteArrayOutputStream()

			def zipStream = new java.util.zip.GZIPOutputStream(targetStream)
			zipStream.write(outputFile.bytes)
			def zipFile = new File("${outputFile.getAbsolutePath()}.gz")
			zipFile.createNewFile()
			zipFile.bytes = targetStream.toByteArray()
			targetStream.close()


		}
		else {
			println("Asset File not found! ${fileName}")
		}
	}

	// Save File
	// Save File Digest
	// Gzip File
	// Update Manifest
}

setDefaultTarget(assetPrecompile)
<commit_msg>added gzip directives and cleanup of processing<commit_after>import org.codehaus.groovy.grails.commons.ApplicationHolder
import org.apache.tools.ant.DirectoryScanner
// import asset.pipeline.*;

includeTargets << grailsScript("_GrailsBootstrap")

target(assetPrecompile: "Precompiles assets in the application as specified by the precompile glob!") {
    depends(configureProxy,compile, packageApp, bootstrap)

    def assetHelper             = classLoader.loadClass('asset.pipeline.AssetHelper')
    def directiveProcessorClass = classLoader.loadClass('asset.pipeline.DirectiveProcessor')
    def uglifyJsProcessor 		= classLoader.loadClass('asset.pipeline.processors.UglifyJsProcessor').newInstance()
	def grailsApplication       = ApplicationHolder.getApplication()
    event("StatusUpdate",["Precompiling Assets!"]);

    // Clear compiled assets folder
    def assetDir = new File("web-app/assets")
    if(assetDir.exists()) {
    	assetDir.deleteDir()
    }


	//Find all files we want to process
	DirectoryScanner scanner = new DirectoryScanner()
	scanner.setExcludes(["**/.*","**/.DS_Store"] as String[])
	scanner.setIncludes(["**/*"] as String[])

	def assetPaths = assetHelper.getAssetPaths()
	def filesToProcess = []
	for(path in assetPaths) {
	    event("StatusUpdate",["Scanning ${path}"]);
	    scanner.setBasedir(path)
	    scanner.setCaseSensitive(false)
	    scanner.scan()
	    filesToProcess += scanner.getIncludedFiles().flatten()
	}
	filesToProcess.unique()

	for(counter = 0 ; counter < filesToProcess.size(); counter++) {
		def fileName = filesToProcess[counter]
		event("StatusUpdate",["Processing File ${counter+1} of ${filesToProcess.size()} - ${fileName}"]);
		def extension = assetHelper.extensionFromURI(fileName)
		fileName = fileName.substring(0,fileName.lastIndexOf("."))

		def assetFile = assetHelper.artefactForFileWithExtension(assetHelper.fileForUri(fileName,null,extension), extension)
		if(assetFile) {
			def fileData = null

			if(assetFile.class.name == 'java.io.File') {
				// println "Non Asset File"
			}
			else {
				def directiveProcessor = directiveProcessorClass.newInstance(assetFile.contentType)
				fileData = directiveProcessor.compile(assetFile)
				if(assetFile.contentType == 'application/javascript') {
					def newFileData = fileData
					try {
						event("StatusUpdate",["Uglifying File ${counter+1} of ${filesToProcess.size()} - ${fileName}"]);
						newFileData = uglifyJsProcessor.process(fileData)
					} catch(e) {
						println "Uglify JS Exception ${e}"
						newFileData = fileData
					}
					fileData = newFileData
				}
			}


			def outputFile = new File("web-app/assets/${fileName}.${extension}");
			def parentTree = new File(outputFile.parent)
			parentTree.mkdirs()
			outputFile.createNewFile();
			if(fileData) {
				outputFile.text = fileData
			} else {
				if(assetFile.class.name == 'java.io.File') {
					outputFile.bytes = assetFile.bytes
				} else {
					outputFile.bytes = assetFile.file.bytes
				}
			}
			def targetStream = new java.io.ByteArrayOutputStream()

			def zipStream = new java.util.zip.GZIPOutputStream(targetStream)
			event("StatusUpdate",["Compressing File ${counter+1} of ${filesToProcess.size()} - ${fileName}"]);
			zipStream.write(outputFile.bytes)
			def zipFile = new File("${outputFile.getAbsolutePath()}.gz")
			zipFile.createNewFile()
			zipStream.finish()
			zipFile.bytes = targetStream.toByteArray()
			targetStream.close()


		}
		else {
			println("Asset File not found! ${fileName}")
		}
	}

	// Save File Digest
	// Update Manifest
}

setDefaultTarget(assetPrecompile)

{'fraction_non_alphanumeric': 0.12780630961115186, 'fraction_numerical': 0.001173881144534116, 'mean_word_length': 4.231005372217958, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2465410', 'n_tokens_mistral': 2331, 'n_tokens_neox': 2050, 'n_words': 388}