<commit_before>/*
 * bq.com
 *
 * @author <NAME> (<EMAIL>)
 * @date 15/6/16
 * Copyright. All Rights Reserved.
 */
package org.hs5tb.groospin.common;

class Ini {

    static String defaultSection = "#"

    Ini parent
    Map sections = [:]

    Ini putAll(Map map) {
        return putAll(defaultSection, map)
    }

    Ini putAll(String section, Map map) {
        map.each { key, val ->
            put(section, key, val)
        }
        return this
    }

    Ini parse(String iniRaw, String section = null, Collection<String> keys = null) {
        return parse(new StringReader(iniRaw), section, keys)
    }

    Ini parse(File iniFile, String section = null, Collection<String> keys = null) {
        return iniFile.exists() ? parse(iniFile.newReader(), section, keys) : this
    }
    /*
    RocketLauncher INI policy is:
    Ignore duplicated sections (only the first section is used)
    Ignore duplicated keys in the same section (only the first key is used)
     */
    static boolean ignoreDuplicatedSections = true
    static boolean ignoreDuplicatedKeys = true
    Ini parse(Reader iniFile, String section = null, Collection<String> includeOnlyKeys = null) {
        Set<String> sectionsParsed = new HashSet<>()
        Set<String> keysParsed  = new HashSet<>()
        boolean ignoreDuplicatedSection = false
        String currentSection = Ini.defaultSection
        section = section?.trim() ? canonical(section) : null
        Ini ini = new Ini()
        includeOnlyKeys = includeOnlyKeys ? includeOnlyKeys.collect { String k -> canonical(k) } : null
        iniFile.eachLine { String line ->
            line = line.trim()
            if (!line || line.startsWith("#") || line.startsWith(";")) {
                return // Ignore comments
            } else if (line.startsWith("[")) {
                int pos = line.indexOf("]")
                if (pos > 1) {
                    currentSection = canonical(line.substring(1, pos))
                    keysParsed.clear()
                    if (currentSection in sectionsParsed) {
                        ignoreDuplicatedSection = true
                    } else {
                        if (ignoreDuplicatedSections) sectionsParsed << currentSection
                        ignoreDuplicatedSection = false
                    }
                }
            } else {
                if (line.contains("=") && !ignoreDuplicatedSection && (section == null || section == currentSection)) {
                    int equalsPos = line.indexOf("=")
                    String key = canonical(line.substring(0, equalsPos))
                    if (!keysParsed.contains(key) && (!includeOnlyKeys || key in includeOnlyKeys)) {
                        if (ignoreDuplicatedKeys) keysParsed << key
                        String value = line.substring(equalsPos + 1).trim()
                        ini.put(currentSection, key, value)
                    }
                }
            }
        }
        return ini<commit_msg>allow inis to read mame category files (without values)<commit_after>/*
 * bq.com
 *
 * @author <NAME> (<EMAIL>)
 * @date 15/6/16
 * Copyright. All Rights Reserved.
 */
package org.hs5tb.groospin.common;

class Ini {

    static String defaultSection = "#"

    Ini parent
    Map sections = [:]

    Ini putAll(Map map) {
        return putAll(defaultSection, map)
    }

    Ini putAll(String section, Map map) {
        map.each { key, val ->
            put(section, key, val)
        }
        return this
    }

    Ini parseText(String iniRaw, String sectionToParse = null, Collection<String> keysToParse = null) {
        return parse(new StringReader(iniRaw), sectionToParse, keysToParse)
    }

    Ini parse(String iniFile, String sectionToParse = null, Collection<String> keys = null) {
        parse(new File(iniFile), sectionToParse, keys)
    }
    Ini parse(File iniFile, String sectionToParse = null, Collection<String> keysToParse = null) {
        return iniFile.exists() ? parse(iniFile.newReader(), sectionToParse, keysToParse) : this
    }
    /*
    RocketLauncher INI policy is:
    Ignore duplicated sections (only the first section is used)
    Ignore duplicated keys in the same section (only the first key is used)
     */
    static boolean ignoreDuplicatedSections = true
    static boolean ignoreDuplicatedKeys = true
    Ini parse(Reader iniFile, String sectionToParse = null, Collection<String> keysToParse = null) {
        Set<String> sectionsParsed = new HashSet<>()
        Set<String> keysParsed  = new HashSet<>()
        boolean ignoreDuplicatedSection = false
        String currentSection = Ini.defaultSection
        sectionToParse = sectionToParse?.trim() ? canonical(sectionToParse) : null
        Ini ini = new Ini()
        keysToParse = keysToParse ? keysToParse.collect { String k -> canonical(k) } : null
        iniFile.eachLine { String line ->
            line = line.trim()
            if (!line || line.startsWith("#") || line.startsWith(";")) {
                return // Ignore comments
            } else if (line.startsWith("[")) {
                int pos = line.indexOf("]")
                if (pos > 1) {
                    currentSection = canonical(line.substring(1, pos))
                    keysParsed.clear()
                    if (currentSection in sectionsParsed) {
                        ignoreDuplicatedSection = true
                    } else {
                        if (ignoreDuplicatedSections) sectionsParsed << currentSection
                        ignoreDuplicatedSection = false
                    }
                }
            } else {
                if (!ignoreDuplicatedSection && (sectionToParse == null || sectionToParse == currentSection)) {
                    String key
                    String value
                    if (line.contains("=")) {
                        int equalsPos = line.indexOf("=")
                        key = canonical(line.substring(0, equalsPos))
                        value = line.substring(equalsPos + 1).trim()
                    } else {
                        key = canonical(line)
                        value = null
                    }
                    if (!keysParsed.contains(key) && (!keysToParse || key in keysToParse)) {
                        if (ignoreDuplicatedKeys) keysParsed << key
                        ini.put(currentSection, key, value)
                    }
                }
            }
        }
        return ini
{'fraction_non_alphanumeric': 0.09803618370187103, 'fraction_numerical': 0.003092624091541673, 'mean_word_length': 1.793952483801296, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3309701', 'n_tokens_mistral': 1717, 'n_tokens_neox': 1621, 'n_words': 456}