<commit_before>package net.milosvasic.decorator

import net.milosvasic.decorator.content.Messages
import net.milosvasic.decorator.data.Collection
import net.milosvasic.decorator.data.Data
import net.milosvasic.decorator.data.TemplateData
import net.milosvasic.decorator.data.Value
import net.milosvasic.decorator.data.state.ElseState
import net.milosvasic.decorator.data.state.ForeachState
import net.milosvasic.decorator.data.state.IfState
import net.milosvasic.decorator.separator.Separator
import net.milosvasic.decorator.template.TemplateSystem
import net.milosvasic.logger.SimpleLogger
import net.milosvasic.logger.VariantsConfiguration
import net.milosvasic.tautology.Tautology
import net.milosvasic.tautology.expression.ExpressionValue
import net.milosvasic.tautology.parser.TautologyParser
import net.milosvasic.tautology.parser.TautologyParserDelegate
import java.lang.IllegalStateException
import java.util.regex.Pattern

class Decorator : TemplateSystem {

    override val tags = DecoratorTags()
    override val templateExtension = "decoration"
    override val memberSeparator = Separator.MEMBER()
    override val templateMainClass = DecoratorTemplateClass()

    private val tautology = Tautology()
    private val logger = SimpleLogger(VariantsConfiguration(BuildConfig.VARIANT, listOf("DEV")))

    override fun decorate(template: String, data: Data): String {
        val templateFile = javaClass.classLoader.getResource("$template.$templateExtension")
        val rendered = StringBuilder()
        val content = templateFile.readText()
        val rows = mutableListOf<String>()
        rows.addAll(content.split("\n"))
        var ifState: IfState? = null
        val ifStates = mutableListOf<IfState?>()
        var elseState: ElseState? = null
        val elseStates = mutableListOf<ElseState?>()
        var foreachState: ForeachState? = null
        val foreachStates = mutableListOf<ForeachState?>()
        val rowsToBeIgnored = mutableListOf<Int>()
        val foreachTemplates = mutableMapOf<Int, String>()
        val decoratedRows = mutableMapOf<Int, List<String>>()

        rows.forEachIndexed {
            index, line ->
            var row = line

            // Trim comments that are not at the line start
            val lineCommentIndex = row.indexOf(tags.lineComment)
            if (!row.startsWith(tags.lineComment) && lineCommentIndex > 0) {
                row = row.substring(0, lineCommentIndex)
            }
            rows[index] = row

            // Trim tab placeholder
            row = row.replace(tags.tabPlaceholder, "")
            rows[index] = row

            // Parse <include> tags
            val pInclude = Pattern.compile("${tags.includeOpen}(.+?)${tags.includeClose}")
            val mInclude = pInclude.matcher(line)
            while (mInclude.find()) {
                val include = mInclude.group(1)
                var element = decorate(include, data)
                if (element.endsWith("\n")) {
                    element = element.substring(0, element.lastIndex)
                }
                row = row.replace(mInclude.group(0), element)
                rows[index] = row
            }
        }

        rows.forEachIndexed {
            index, line ->
            var row = line
            // Parse <foreach>
            val pFor = Pattern.compile("${tags.foreachOpen}(.+?)${tags.foreachClose}")
            val mFor = pFor.matcher(line)
            while (mFor.find()) {
                val forCondition = mFor.group(1)
                row = row.replace(mFor.group(0), "")
                if (!row.isEmpty()) {
                    throw IllegalStateException(Messages.CONTENT_AFTER_FOR_OPENING(template, index))
                }
                rows[index] = row
                if (foreachState != null) {
                    throw IllegalStateException(Messages.FOR_NOT_CLOSED(template, index))
                } else {
                    foreachState = ForeachState(index, -1, forCondition)
                }
            }

            // Parse <endfor/>
            val pEndfor = Pattern.compile(tags.endFor)
            val mEndfor = pEndfor.matcher(line)
            while (mEndfor.find()) {
                row = row.replace(mEndfor.group(0), "")
                if (row.isEmpty()) {
                    rowsToBeIgnored.add(index)
                }
                rows[index] = row
                if (foreachState == null) {
                    throw IllegalStateException(Messages.FOR_NOT_OPENED(template, index))
                } else {
                    foreachState?.to = index
                    foreachStates.add(foreachState)
                    foreachState = null
                }
            }

            var foreachStateInProgress = false
            val currentState = foreachState
            currentState?.let {
                foreachStateInProgress = index > currentState.from
            }
            if (foreachStateInProgress) {
                foreachTemplates[index] = line
                rowsToBeIgnored.add(index)
            }
        }

        rows.forEachIndexed {
            index, line ->
            var row = line
            val state = getForeachState(foreachStates, index)
            state?.let {
                if (state.from == index) {
                    val templateRows = mutableListOf<String>()
                    for (x in state.from..state.to) {
                        foreachTemplates[x]?.let {
                            item ->
                            templateRows.add(item)
                        }
                    }
                    row = resolveForeach(template, index, templateRows, data, state.value)
                }
            }
            rows[index] = row
        }

        rows.forEachIndexed {
            index, line ->
            var row = line

            // Parse <if> tags
            val pIf = Pattern.compile("${tags.ifOpen}(.+?)${tags.ifClose}")
            val mIf = pIf.matcher(line)
            while (mIf.find()) {
                val ifCondition = mIf.group(1)
                val result = resolveIf(template, data, ifCondition, index)
                row = row.replace(mIf.group(0), "")
                if (row.isEmpty()) {
                    rowsToBeIgnored.add(index)
                }
                rows[index] = row
                if (ifState != null) {
                    throw IllegalStateException(Messages.IF_NOT_CLOSED(template, index))
                } else {
                    ifState = IfState(index, -1, result)
                }
            }

            // Parse <else> tags
            val pElse = Pattern.compile(tags.elseTag)
            val mElse = pElse.matcher(line)
            while (mElse.find()) {
                row = row.replace(mElse.group(0), "")
                if (row.isEmpty()) {
                    rowsToBeIgnored.add(index)
                }
                rows[index] = row
                if (ifState == null) {
                    throw IllegalStateException(Messages.IF_NOT_OPENED(template, index))
                } else {
                    if (elseState != null) {
                        throw IllegalStateException(Messages.ELSE_NOT_CLOSED(template, index))
                    } else {
                        elseState = ElseState(index, -1)
                    }
                }
            }

            // Parse <endif/> tag
            val pEndIf = Pattern.compile(tags.endIf)
            val mEndIf = pEndIf.matcher(line)
            while (mEndIf.find()) {
                row = row.replace(mEndIf.group(0), "")
                if (row.isEmpty()) {
                    rowsToBeIgnored.add(index)
                }
                rows[index] = row
                if (ifState != null) {
                    ifState?.to = index
                    ifStates.add(ifState)
                    ifState = null
                } else {
                    throw IllegalStateException(Messages.IF_NOT_OPENED(template, index))
                }
                if (elseState != null) {
                    elseState?.to = index
                    elseStates.add(elseState)
                    elseState = null
                }
            }

            // Parse <dc> tags
            val p = Pattern.compile("${tags.open}(.+?)${tags.close}")
            val m = p.matcher(line)
            val commands = mutableListOf<String>()
            while (m.find()) {
                val result = m.group()
                commands.add(result)
            }
            if (!commands.isEmpty()) {
                decoratedRows[index] = commands
            }
        }

        rows.forEachIndexed {
            index, line ->
            var isLineValid = !line.startsWith(tags.lineComment)
            if (isLineValid && !satisfiesIf(ifStates, index)) {
                isLineValid = satisfiesElse(elseStates, index)
            }
            if (!rowsToBeIgnored.contains(index) && isLineValid) {<commit_msg>Decorator: Working on complex 'foreach'.<commit_after>package net.milosvasic.decorator

import net.milosvasic.decorator.content.Messages
import net.milosvasic.decorator.data.Collection
import net.milosvasic.decorator.data.Data
import net.milosvasic.decorator.data.TemplateData
import net.milosvasic.decorator.data.Value
import net.milosvasic.decorator.data.state.ElseState
import net.milosvasic.decorator.data.state.ForeachState
import net.milosvasic.decorator.data.state.IfState
import net.milosvasic.decorator.separator.Separator
import net.milosvasic.decorator.template.TemplateSystem
import net.milosvasic.logger.SimpleLogger
import net.milosvasic.logger.VariantsConfiguration
import net.milosvasic.tautology.Tautology
import net.milosvasic.tautology.expression.ExpressionValue
import net.milosvasic.tautology.parser.TautologyParser
import net.milosvasic.tautology.parser.TautologyParserDelegate
import java.lang.IllegalStateException
import java.util.regex.Pattern

class Decorator : TemplateSystem {

    override val tags = DecoratorTags()
    override val templateExtension = "decoration"
    override val memberSeparator = Separator.MEMBER()
    override val templateMainClass = DecoratorTemplateClass()

    private val tautology = Tautology()
    private val logger = SimpleLogger(VariantsConfiguration(BuildConfig.VARIANT, listOf("DEV")))

    override fun decorate(template: String, data: Data): String {
        val templateFile = javaClass.classLoader.getResource("$template.$templateExtension")
        val rendered = StringBuilder()
        val content = templateFile.readText()
        val rows = mutableListOf<String>()
        rows.addAll(content.split("\n"))
        var ifState: IfState? = null
        val ifStates = mutableListOf<IfState?>()
        var elseState: ElseState? = null
        val elseStates = mutableListOf<ElseState?>()
        var foreachState: ForeachState? = null
        val foreachStates = mutableListOf<ForeachState?>()
        val rowsToBeIgnored = mutableListOf<Int>()
        val foreachTemplates = mutableMapOf<Int, String>()
        val decoratedRows = mutableMapOf<Int, List<String>>()

        rows.forEachIndexed {
            index, line ->
            var row = line

            // Trim comments that are not at the line start
            val lineCommentIndex = row.indexOf(tags.lineComment)
            if (!row.startsWith(tags.lineComment) && lineCommentIndex > 0) {
                row = row.substring(0, lineCommentIndex)
            }
            rows[index] = row

            // Trim comments that are at line start:
            if (row.startsWith(tags.lineComment)) {
                row = ""
                rows[index] = row
                rowsToBeIgnored.add(index)
            }

            // Trim tab placeholder
            row = row.replace(tags.tabPlaceholder, "")
            rows[index] = row

            // Parse <include> tags
            val pInclude = Pattern.compile("${tags.includeOpen}(.+?)${tags.includeClose}")
            val mInclude = pInclude.matcher(line)
            while (mInclude.find()) {
                val include = mInclude.group(1)
                var element = decorate(include, data)
                if (element.endsWith("\n")) {
                    element = element.substring(0, element.lastIndex)
                }
                row = row.replace(mInclude.group(0), element)
                rows[index] = row
            }
        }

        rows.forEachIndexed {
            index, line ->
            var row = line
            // Parse <foreach>
            val pFor = Pattern.compile("${tags.foreachOpen}(.+?)${tags.foreachClose}")
            val mFor = pFor.matcher(line)
            while (mFor.find()) {
                val forCondition = mFor.group(1)
                row = row.replace(mFor.group(0), "")
                if (!row.isEmpty()) {
                    throw IllegalStateException(Messages.CONTENT_AFTER_FOR_OPENING(template, index))
                }
                rows[index] = row
                if (foreachState != null) {
                    throw IllegalStateException(Messages.FOR_NOT_CLOSED(template, index))
                } else {
                    foreachState = ForeachState(index, -1, forCondition)
                }
            }

            // Parse <endfor/>
            val pEndfor = Pattern.compile(tags.endFor)
            val mEndfor = pEndfor.matcher(line)
            while (mEndfor.find()) {
                row = row.replace(mEndfor.group(0), "")
                if (row.isEmpty()) {
                    rowsToBeIgnored.add(index)
                }
                rows[index] = row
                if (foreachState == null) {
                    throw IllegalStateException(Messages.FOR_NOT_OPENED(template, index))
                } else {
                    foreachState?.to = index
                    foreachStates.add(foreachState)
                    foreachState = null
                }
            }

            var foreachStateInProgress = false
            val currentState = foreachState
            currentState?.let {
                foreachStateInProgress = index > currentState.from
            }
            if (foreachStateInProgress) {
                foreachTemplates[index] = line
                rowsToBeIgnored.add(index)
            }
        }

        rows.forEachIndexed {
            index, line ->
            var row = line
            val state = getForeachState(foreachStates, index)
            state?.let {
                if (state.from == index) {
                    val templateRows = mutableListOf<String>()
                    for (x in state.from..state.to) {
                        foreachTemplates[x]?.let {
                            item ->
                            templateRows.add(item)
                        }
                    }
                    row = resolveForeach(template, index, templateRows, data, state.value)
                }
            }
            rows[index] = row
        }

        rows.forEachIndexed {
            index, line ->
            var row = line

            // Parse <if> tags
            val pIf = Pattern.compile("${tags.ifOpen}(.+?)${tags.ifClose}")
            val mIf = pIf.matcher(line)
            while (mIf.find()) {
                val ifCondition = mIf.group(1)
                val result = resolveIf(template, data, ifCondition, index)
                row = row.replace(mIf.group(0), "")
                if (row.isEmpty()) {
                    rowsToBeIgnored.add(index)
                }
                rows[index] = row
                if (ifState != null) {
                    throw IllegalStateException(Messages.IF_NOT_CLOSED(template, index))
                } else {
                    ifState = IfState(index, -1, result)
                }
            }

            // Parse <else> tags
            val pElse = Pattern.compile(tags.elseTag)
            val mElse = pElse.matcher(line)
            while (mElse.find()) {
                row = row.replace(mElse.group(0), "")
                if (row.isEmpty()) {
                    rowsToBeIgnored.add(index)
                }
                rows[index] = row
                if (ifState == null) {
                    throw IllegalStateException(Messages.IF_NOT_OPENED(template, index))
                } else {
                    if (elseState != null) {
                        throw IllegalStateException(Messages.ELSE_NOT_CLOSED(template, index))
                    } else {
                        elseState = ElseState(index, -1)
                    }
                }
            }

            // Parse <endif/> tag
            val pEndIf = Pattern.compile(tags.endIf)
            val mEndIf = pEndIf.matcher(line)
            while (mEndIf.find()) {
                row = row.replace(mEndIf.group(0), "")
                if (row.isEmpty()) {
                    rowsToBeIgnored.add(index)
                }
                rows[index] = row
                if (ifState != null) {
                    ifState?.to = index
                    ifStates.add(ifState)
                    ifState = null
                } else {
                    throw IllegalStateException(Messages.IF_NOT_OPENED(template, index))
                }
                if (elseState != null) {
                    elseState?.to = index
                    elseStates.add(elseState)
                    elseState = null
                }
            }

            // Parse <dc> tags
            val p = Pattern.compile("${tags.open}(.+?)${tags.close}")
            val m = p.matcher(line)
            val commands = mutableListOf<String>()
            while (m.find()) {
                val result = m.group()
                commands.add(result)
            }
            if (!commands.isEmpty()) {
                decoratedRows[index] = commands
            }
        }

        rows.forEachIndexed {
            index, line ->
            var isLineValid = true
            if (!satisfiesIf(ifStates, index)) {
                isLineValid = satisfiesElse(elseStates, index)
            }
            if (!rowsToBeIgnored.contains(index) && isLineValid) {
{'fraction_non_alphanumeric': 0.09738756613756613, 'fraction_numerical': 0.0016534391534391533, 'mean_word_length': 1.5155968390406211, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '256584', 'n_tokens_mistral': 4842, 'n_tokens_neox': 4615, 'n_words': 916}