<commit_before>import { FS } from 'grind-support'
import './Template'

export class Compiler {
	engine = null

	constructor(engine) {
		this.engine = engine
	}

	async compile(template) {
		let contents = (await FS.readFile(template)).toString().trim()
		let extendsLayout = null

		const expressions = [ ]
		let match = null

		while((match = contents.match(/@([a-z0-9_]+)(\s*\()?/))) {
			if(match.index > 0) {
				expressions.push({
					type: 'string',
					contents: contents.substring(0, match.index)
				})

				contents = contents.substring(match.index)
			}

			let args = null

			if(match[2]) {
				let openCount = -1
				let index = 0
				let startIndex = 0
				let lastIndex = 0

				while(openCount !== 0 && (index = Template.nextIndexOfParenthesis(contents, index)) >= 0) {
					const parenthesis = contents.substring(index, index + 1)

					if(parenthesis === ')') {
						openCount--
					} else if(openCount === -1) {
						startIndex = index
						openCount = 1
					} else {
						openCount++
					}

					lastIndex = index
					index++
				}

				args = contents.substring(startIndex + 1, lastIndex)
				contents = contents.substring(lastIndex + 1)
			} else {
				contents = contents.substring(match[0].length)
			}

			if(match[1] === 'extends') {
				extendsLayout = args
				continue
			}

			const method = `compile${match[1][0].toUpperCase()}${match[1].substring(1)}`
			const result = this[method](args)

			if(!result.isNil) {
				expressions.push({
					type: 'code',
					contents: result
				})
			}
		}

		contents = contents.trim()

		if(contents.length > 0) {
			expressions.push({
				type: 'string',
				contents: contents
			})
		}

		let code = ''

		for(const { type, contents } of expressions) {
			if(type === 'code') {
				code += `${contents}\n`
			} else {
				code += `output += \`${Template.sanitizeHtml(contents)}\`;\n`
			}
		}

		code = `template = function(_, _sections = { }) {\nlet output = '';\n${Template.contextualize(code)}\n`

		if(extendsLayout !== null) {
			code += `return _.$compiler._extends(${extendsLayout}, _, _sections);\n}`
		} else {
			code += `return output;\n}`
		}

		return eval(code)
	}

	_extends(template, context, sections) {
		return this.compile(this.engine.resolve(template)).then(template => template(context, sections))
	}

	compileIf(args) {
		return `if(${args}) {`
	}

	compileElseif(args) {
		return `} else if(${args}) {`
	}

	compileElse() {
		return '} else {'
	}

	compileEndif() {
		return this.compileEnd()
	}

	compileFor(args) {
		return `for(${args}) {`
	}

	compileEndfor() {
		return this.compileEnd()
	}

	compileWhile(args) {
		return `while(${args}) {`
	}

	compileEndwhile() {
		return this.compileEnd()
	}
<commit_msg>Started work on custom directives<commit_after>import { FS } from 'grind-support'
import './Template'

export class Compiler {
	engine = null
	directives = { }

	constructor(engine) {
		this.engine = engine
	}

	async compile(template) {
		let contents = (await FS.readFile(template)).toString().trim()
		let extendsLayout = null

		const expressions = [ ]
		let match = null

		// Loops through and finds all directives
		while((match = contents.match(/@([a-z0-9_]+)(\s*\()?/))) {
			if(match.index > 0) {
				expressions.push({
					type: 'string',
					contents: contents.substring(0, match.index)
				})

				contents = contents.substring(match.index)
			}

			let args = null

			if(match[2]) {
				let openCount = -1
				let index = 0
				let startIndex = 0
				let lastIndex = 0

				while(openCount !== 0 && (index = Template.nextIndexOfParenthesis(contents, index)) >= 0) {
					const parenthesis = contents.substring(index, index + 1)

					if(parenthesis === ')') {
						openCount--
					} else if(openCount === -1) {
						startIndex = index
						openCount = 1
					} else {
						openCount++
					}

					lastIndex = index
					index++
				}

				args = contents.substring(startIndex + 1, lastIndex)
				contents = contents.substring(lastIndex + 1)
			} else {
				contents = contents.substring(match[0].length)
			}

			if(match[1] === 'extends') {
				extendsLayout = args
				continue
			}

			const result = this.compileDirective(match[1], args)

			if(!result.isNil) {
				expressions.push({
					type: 'code',
					contents: result
				})
			}
		}

		contents = contents.trim()

		if(contents.length > 0) {
			expressions.push({
				type: 'string',
				contents: contents
			})
		}

		let code = ''

		for(const { type, contents } of expressions) {
			if(type === 'code') {
				code += `${contents}\n`
			} else {
				code += `output += \`${Template.sanitizeHtml(contents)}\`;\n`
			}
		}

		code = `template = function(_, _sections = { }) {\nlet output = '';\n${Template.contextualize(code)}\n`

		if(extendsLayout !== null) {
			code += `return _.$compiler._extends(${extendsLayout}, _, _sections);\n}`
		} else {
			code += `return output;\n}`
		}

		return eval(code)
	}

	_extends(template, context, sections) {
		return this.compile(this.engine.resolve(template)).then(template => template(context, sections))
	}

	compileDirective(name, args) {
		if(name === 'directive') {
			// Avoid infinite loop
			return null
		}

		if(typeof this.directives[name] === 'function') {
			return this.directives[name](args)
		}

		return this[`compile${name[0].toUpperCase()}${name.substring(1)}`](args)
	}

	compileIf(condition) {
		return `if(${condition}) {`
	}

	compileElseif(condition) {
		return `} else if(${condition}) {`
	}

	compileElse() {
		return '} else {'
	}

	compileEndif() {
		return this.compileEnd()
	}

	compileFor(args) {
		return `for(${args}) {`
	}

	compileEndfor() {
		return this.compileEnd()
	}

	compileWhile(condition) {
		return `while(${condition}) {`
	}

	compileEndwhile() {
		return this.compileEnd()
	}

{'fraction_non_alphanumeric': 0.1653393351800554, 'fraction_numerical': 0.007790858725761773, 'mean_word_length': 2.99239806496199, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3776562', 'n_tokens_mistral': 2403, 'n_tokens_neox': 1927, 'n_words': 391}