<commit_before>package ksax

import org.xml.sax.Attributes
import org.xml.sax.helpers.DefaultHandler
import java.io.InputStream
import java.util.*
import javax.xml.parsers.SAXParserFactory

class KSAXRule<out T : Any>(val path: String, val converter: (String) -> Pair<String, T>)

class KSAXPostProcessRule<out T : Any>(val path: String, val converter: (HashMap<String, Any>) -> Pair<String, T>)

interface KSAXRuleBuilder {
    fun <T : Any> push(pathToName: Pair<String, String>, converter: (String) -> T)
    fun push(pathToName: Pair<String, String>)
    fun push(tagName: String)
    fun <T : Any> pushToList(pathToName: Pair<String, String>, converter: (HashMap<String, Any>) -> T)
}

class KSAXRuleBuilderImpl : KSAXRuleBuilder {

    private val internalNodeRules: HashMap<String, KSAXRule<Any>> = HashMap()
    private val internalAttrRules: HashMap<String, HashMap<String, KSAXRule<Any>>> = HashMap()
    private val internalPostProcessRules = HashMap<String, KSAXPostProcessRule<Any>>()

    private val internalAllRules = ArrayList<String>()

    val allRules: List<String>
        get() = internalAllRules

    val nodeRules: Map<String, KSAXRule<Any>>
        get() = internalNodeRules

    val attrRules: Map<String, Map<String, KSAXRule<*>>>
        get() = internalAttrRules

    val postProcessRules: Map<String, KSAXPostProcessRule<*>>
        get() = internalPostProcessRules

    override fun <T : Any> push(pathToName: Pair<String, String>, converter: (String) -> T) {
        val f: (String) -> Pair<String, T> = {
            pathToName.second to converter(it)
        }

        val rule = KSAXRule(pathToName.first, f)

        internalAllRules.add(pathToName.first)

        if (rule.path.contains('@')) {
            val node = rule.path.substringBefore('@')
            val attr = rule.path.substringAfter('@')
            internalAttrRules.putIfAbsent(node, HashMap())
            val attrRules = internalAttrRules[node]
            attrRules!![attr] = rule
        } else {
            internalNodeRules[rule.path] = rule
        }
    }

    override fun push(pathToName: Pair<String, String>) {
        val f: (String) -> String = { it }
        push(pathToName, f)
    }

    override fun push(tagName: String) = push(tagName to tagName)

    override fun <T : Any> pushToList(pathToName: Pair<String, String>, converter: (HashMap<String, Any>) -> T) {
        val f: (HashMap<String, Any>) -> Pair<String, T> = {
            pathToName.second to converter(it)
        }
        val rule = KSAXPostProcessRule(pathToName.first, f)
        internalAllRules.add(pathToName.first)
        internalPostProcessRules[pathToName.first] = rule
    }
}

fun HashMap<String, Any>.pop(key: String): String? = run {
    val result = remove(key)
    result as String?
}

fun HashMap<String, Any>.popNonNull(key: String): String = this.pop(key)!!<commit_msg>Ability to mark rule as optional.<commit_after>package ksax

import org.xml.sax.Attributes
import org.xml.sax.helpers.DefaultHandler
import java.io.InputStream
import java.util.*
import javax.xml.parsers.SAXParserFactory

class KSAXRule<out T : Any>(val path: String, val converter: (String) -> Pair<String, T>)

class KSAXPostProcessRule<out T : Any>(val path: String, val converter: (HashMap<String, Any>) -> Pair<String, T>)

interface KSAXRuleBuilder {
    fun <T : Any> push(pathToName: Pair<String, String>, optional: Boolean = false, converter: (String) -> T)
    fun push(pathToName: Pair<String, String>, optional: Boolean = false)
    fun push(tagName: String, optional: Boolean = false)
    fun <T : Any> pushToList(pathToName: Pair<String, String>, optional: Boolean = false, converter: (HashMap<String, Any>) -> T)
}

class KSAXRuleBuilderImpl : KSAXRuleBuilder {

    private val internalNodeRules: HashMap<String, KSAXRule<Any>> = HashMap()
    private val internalAttrRules: HashMap<String, HashMap<String, KSAXRule<Any>>> = HashMap()
    private val internalPostProcessRules = HashMap<String, KSAXPostProcessRule<Any>>()

    private val internalAllRequiredRules = ArrayList<String>()

    val allRules: List<String>
        get() = internalAllRequiredRules

    val nodeRules: Map<String, KSAXRule<Any>>
        get() = internalNodeRules

    val attrRules: Map<String, Map<String, KSAXRule<*>>>
        get() = internalAttrRules

    val postProcessRules: Map<String, KSAXPostProcessRule<*>>
        get() = internalPostProcessRules

    override fun <T : Any> push(pathToName: Pair<String, String>, optional: Boolean, converter: (String) -> T) {
        val f: (String) -> Pair<String, T> = {
            pathToName.second to converter(it)
        }

        val rule = KSAXRule(pathToName.first, f)

        if (!optional) internalAllRequiredRules.add(pathToName.first)

        if (rule.path.contains('@')) {
            val node = rule.path.substringBefore('@')
            val attr = rule.path.substringAfter('@')
            internalAttrRules.putIfAbsent(node, HashMap())
            val attrRules = internalAttrRules[node]
            attrRules!![attr] = rule
        } else {
            internalNodeRules[rule.path] = rule
        }
    }

    override fun push(pathToName: Pair<String, String>, optional: Boolean) {
        val f: (String) -> String = { it }
        push(pathToName, optional, f)
    }

    override fun push(tagName: String, optional: Boolean) = push(tagName to tagName, optional)

    override fun <T : Any> pushToList(pathToName: Pair<String, String>, optional: Boolean, converter: (HashMap<String, Any>) -> T) {
        val f: (HashMap<String, Any>) -> Pair<String, T> = {
            pathToName.second to converter(it)
        }
        val rule = KSAXPostProcessRule(pathToName.first, f)
        if (!optional) internalAllRequiredRules.add(pathToName.first)
        internalPostProcessRules[pathToName.first] = rule
    }
}

fun HashMap<String, Any>.pop(key: String): String? = run {
    val result = remove(key)
    result as String?
}

fun HashMap<String, Any>.popNonNull(key: String): String = this.pop(key)!!
{'fraction_non_alphanumeric': 0.12462760675273088, 'fraction_numerical': 0.0, 'mean_word_length': 3.427106227106227, 'pattern_counts': {'":': 0, '<': 79, '<?xml version=': 0, '>': 97, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4200576', 'n_tokens_mistral': 1875, 'n_tokens_neox': 1824, 'n_words': 464}