<commit_before>package com.redhat.ceylon.common.config;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.LineNumberReader;
import java.io.PushbackReader;
import java.io.Reader;
import java.nio.CharBuffer;
import java.nio.charset.Charset;
import java.util.InvalidPropertiesFormatException;

public class ConfigReader {
    private ConfigReaderListener listener;
    private InputStream in;
    private LineNumberReader counterdr;
    private MemoPushbackReader reader;
    private String section;
    
    private enum Token { section, option, assign, comment, eol, error, eof }
    
    public ConfigReader(InputStream in, ConfigReaderListener listener) {
        this.in = in;
        this.listener = listener;
    }
    
    public void process() throws IOException {
        section = null;
        try {
            counterdr = new LineNumberReader(new BufferedReader(new InputStreamReader(in, Charset.forName("UTF-8"))));
            reader = new MemoPushbackReader(counterdr);

            listener.setup();
            
            Token tok;
            skipWhitespace(true);
            flushWhitespace();
            while ((tok = peekToken()) != Token.eof) {
                switch (tok) {
                case section:
                    handleSection();
                    break;
                case option:
                    if (section != null) {
                        handleOption();
                    } else {
                        throw new InvalidPropertiesFormatException("Option without section in configuration file at line " + (counterdr.getLineNumber() + 1));
                    }
                    break;
                case comment:
                    skipToNextLine();
                    listener.onComment(reader.getAndClearMemo());
                    break;
                case eol:
                    skipToNextLine();
                    listener.onWhitespace(reader.getAndClearMemo());
                    break;
                default:
                    throw new InvalidPropertiesFormatException("Unexpected token in configuration file at line " + (counterdr.getLineNumber() + 1));
                }
                skipWhitespace(true);
                flushWhitespace();
            }
            
            listener.cleanup();
        } finally {
            if (reader != null) {
                reader.close();
            }
        }
    }

    private void handleSection() throws IOException {
        expect('[');
        section = readName(true).toLowerCase();
        if (!section.matches("[\\p{L}\\p{Nd}]+(\\.[\\p{L}\\p{Nd}]+)*")) {
            throw new InvalidPropertiesFormatException("Invalid section name in configuration file at line " + (counterdr.getLineNumber() + 1));
        }
        skipWhitespace(false);
        if (reader.peek() == '\"') {
            String subSection = readString();<commit_msg>Section names are no longer converted to lower case just to mimic a deprecated git-config feature<commit_after>package com.redhat.ceylon.common.config;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.LineNumberReader;
import java.io.PushbackReader;
import java.io.Reader;
import java.nio.CharBuffer;
import java.nio.charset.Charset;
import java.util.InvalidPropertiesFormatException;

public class ConfigReader {
    private ConfigReaderListener listener;
    private InputStream in;
    private LineNumberReader counterdr;
    private MemoPushbackReader reader;
    private String section;
    
    private enum Token { section, option, assign, comment, eol, error, eof }
    
    public ConfigReader(InputStream in, ConfigReaderListener listener) {
        this.in = in;
        this.listener = listener;
    }
    
    public void process() throws IOException {
        section = null;
        try {
            counterdr = new LineNumberReader(new BufferedReader(new InputStreamReader(in, Charset.forName("UTF-8"))));
            reader = new MemoPushbackReader(counterdr);

            listener.setup();
            
            Token tok;
            skipWhitespace(true);
            flushWhitespace();
            while ((tok = peekToken()) != Token.eof) {
                switch (tok) {
                case section:
                    handleSection();
                    break;
                case option:
                    if (section != null) {
                        handleOption();
                    } else {
                        throw new InvalidPropertiesFormatException("Option without section in configuration file at line " + (counterdr.getLineNumber() + 1));
                    }
                    break;
                case comment:
                    skipToNextLine();
                    listener.onComment(reader.getAndClearMemo());
                    break;
                case eol:
                    skipToNextLine();
                    listener.onWhitespace(reader.getAndClearMemo());
                    break;
                default:
                    throw new InvalidPropertiesFormatException("Unexpected token in configuration file at line " + (counterdr.getLineNumber() + 1));
                }
                skipWhitespace(true);
                flushWhitespace();
            }
            
            listener.cleanup();
        } finally {
            if (reader != null) {
                reader.close();
            }
        }
    }

    private void handleSection() throws IOException {
        expect('[');
        section = readName(true);
        if (!section.matches("[\\p{L}\\p{Nd}]+(\\.[\\p{L}\\p{Nd}]+)*")) {
            throw new InvalidPropertiesFormatException("Invalid section name in configuration file at line " + (counterdr.getLineNumber() + 1));
        }
        skipWhitespace(false);
        if (reader.peek() == '\"') {
            String subSection = readString();
{'fraction_non_alphanumeric': 0.0968063872255489, 'fraction_numerical': 0.0013306719893546241, 'mean_word_length': 1.765869365225391, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1884597', 'n_tokens_mistral': 1484, 'n_tokens_neox': 1423, 'n_words': 352}