<commit_before>/*
 * Copyright 2016 <NAME>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import Foundation

// MARK: Parse

class Parser {
    var keyPath: [String] = []
    var currentKey: String = "."
    var toml: Toml = Toml()

    // MARK: Initializers

    convenience init(toml: Toml) {
        self.init()
        self.toml = toml
    }

    // MARK: Parsing

    public func parse(string: String) throws {
        // Convert input into tokens
        let lexer = Lexer(input: string, grammar: Grammar().grammar)
        let tokens = try lexer.tokenize()
        try parse(tokens: tokens)
    }

    /**
        Parse a TOML token stream construct a dictionary.

        - Parameter tokens: Token stream describing a TOML data structure
    */
    private func parse(tokens: [Token]) throws {
        // A dispatch table for parsing TOML tables
        let TokenMap: [Token: (Token, inout [Token]) throws -> ()] = [
            .Identifier("1"): setValue,
            .IntegerNumber(1): setValue,
            .DoubleNumber(1.0): setValue,
            .Boolean(true): setValue,
            .DateTime(Date()): setValue,
            .TableBegin: setTable,
            .ArrayBegin: setArray,
            .TableArrayBegin: setTableArray,
            .InlineTableBegin: setInlineTable
        ]

        // Convert tokens to values in the Toml
        var myTokens = tokens

        while myTokens.count > 0 {
            let token = myTokens.remove(at: 0)
            if case .Key(let val) = token {
                currentKey = val
            } else {
                try TokenMap[token]!(token, &myTokens)
            }
        }
    }

    /**
        Given a TOML token stream construct an array.

        - Parameter tokens: Token stream describing array

        - Returns: Array populated with values from token stream
    */
    private func parse(tokens: inout [Token]) throws -> [Any] {
        var array: [Any] = [Any]()

        while tokens.count > 0 {
            let token = tokens.remove(at: 0)
            switch token {
                case .Identifier(let val):
                    array.append(val)
                case .IntegerNumber(let val):
                    array.append(val)
                case .DoubleNumber(let val):
                    array.append(val)
                case .Boolean(let val):
                    array.append(val)
                case .DateTime(let val):
                    array.append(val)
                case .InlineTableBegin:
                    array.append(try processInlineTable(tokens: &tokens))
                case .ArrayBegin:
                    var wrap = ArrayWrapper(array: array)
                    try checkAndSetArray(check: parse(tokens: &tokens), key: [""], out: &wrap)
                    array = wrap.array
                default:
                    return array
            }
        }

        return array
    }

    private func processInlineTable(tokens: inout [Token]) throws -> Toml {
        let tableTokens = extractTableTokens(tokens: &tokens, inline: true)
        let tableParser = Parser()
        try tableParser.parse(tokens: tableTokens)
        return tableParser.toml
    }

    /**
        Given a value token set it's value in the `table`

        - Parameter currToken: A value token that is currently being parsed
        - Parameter tokens: Array of remaining tokens in the stream
    */
    private func setValue(currToken: Token, tokens: inout [Token]) throws {
        var key = keyPath
        key.append(currentKey)

        if toml.hasKey(key) {
            throw TomlError.DuplicateKey(String(key))
        }

        toml.setValue(key: key, value: currToken.value)
    }<commit_msg>Fix grammar error<commit_after>/*
 * Copyright 2016 <NAME>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import Foundation

// MARK: Parse

class Parser {
    var keyPath: [String] = []
    var currentKey: String = "."
    var toml: Toml = Toml()

    // MARK: Initializers

    convenience init(toml: Toml) {
        self.init()
        self.toml = toml
    }

    // MARK: Parsing

    public func parse(string: String) throws {
        // Convert input into tokens
        let lexer = Lexer(input: string, grammar: Grammar().grammar)
        let tokens = try lexer.tokenize()
        try parse(tokens: tokens)
    }

    /**
        Parse a TOML token stream construct a dictionary.

        - Parameter tokens: Token stream describing a TOML data structure
    */
    private func parse(tokens: [Token]) throws {
        // A dispatch table for parsing TOML tables
        let TokenMap: [Token: (Token, inout [Token]) throws -> ()] = [
            .Identifier("1"): setValue,
            .IntegerNumber(1): setValue,
            .DoubleNumber(1.0): setValue,
            .Boolean(true): setValue,
            .DateTime(Date()): setValue,
            .TableBegin: setTable,
            .ArrayBegin: setArray,
            .TableArrayBegin: setTableArray,
            .InlineTableBegin: setInlineTable
        ]

        // Convert tokens to values in the Toml
        var myTokens = tokens

        while myTokens.count > 0 {
            let token = myTokens.remove(at: 0)
            if case .Key(let val) = token {
                currentKey = val
            } else {
                try TokenMap[token]!(token, &myTokens)
            }
        }
    }

    /**
        Given a TOML token stream construct an array.

        - Parameter tokens: Token stream describing array

        - Returns: Array populated with values from token stream
    */
    private func parse(tokens: inout [Token]) throws -> [Any] {
        var array: [Any] = [Any]()

        while tokens.count > 0 {
            let token = tokens.remove(at: 0)
            switch token {
                case .Identifier(let val):
                    array.append(val)
                case .IntegerNumber(let val):
                    array.append(val)
                case .DoubleNumber(let val):
                    array.append(val)
                case .Boolean(let val):
                    array.append(val)
                case .DateTime(let val):
                    array.append(val)
                case .InlineTableBegin:
                    array.append(try processInlineTable(tokens: &tokens))
                case .ArrayBegin:
                    var wrap = ArrayWrapper(array: array)
                    try checkAndSetArray(check: parse(tokens: &tokens), key: [""], out: &wrap)
                    array = wrap.array
                default:
                    return array
            }
        }

        return array
    }

    private func processInlineTable(tokens: inout [Token]) throws -> Toml {
        let tableTokens = extractTableTokens(tokens: &tokens, inline: true)
        let tableParser = Parser()
        try tableParser.parse(tokens: tableTokens)
        return tableParser.toml
    }

    /**
        Given a value token set its value in the `table`

        - Parameter currToken: A value token that is currently being parsed
        - Parameter tokens: Array of remaining tokens in the stream
    */
    private func setValue(currToken: Token, tokens: inout [Token]) throws {
        var key = keyPath
        key.append(currentKey)

        if toml.hasKey(key) {
            throw TomlError.DuplicateKey(String(key))
        }

        toml.setValue(key: key, value: currToken.value)
    }
{'fraction_non_alphanumeric': 0.09030497211344488, 'fraction_numerical': 0.0037973181440607573, 'mean_word_length': 1.8009305417082087, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6918207', 'n_tokens_mistral': 2244, 'n_tokens_neox': 2172, 'n_words': 740}