<commit_before>package com.github.tmarsteel.ktprolog.parser.lexer

import com.github.tmarsteel.ktprolog.parser.sequence.IteratorBasedTransactionalSequence
import com.github.tmarsteel.ktprolog.parser.sequence.TransactionalSequence
import com.github.tmarsteel.ktprolog.parser.source.SourceLocation
import com.github.tmarsteel.ktprolog.parser.source.SourceLocationRange
import com.github.tmarsteel.ktprolog.parser.source.SourceUnit

class Lexer(source: Iterator<Char>, initialSourceLocation: SourceLocation) : TransactionalSequence<Token>
{
    private val lexerIteratorTxSequence = IteratorBasedTransactionalSequence(LexerIterator(source, initialSourceLocation))

    constructor(unit: SourceUnit, source: Iterator<Char>): this(source, SourceLocation(unit, 1, 0, 0))

    override fun mark() = lexerIteratorTxSequence.mark()

    override fun commit() = lexerIteratorTxSequence.commit()

    override fun hasNext() = lexerIteratorTxSequence.hasNext()

    override fun next() = lexerIteratorTxSequence.next()

    override fun rollback() = lexerIteratorTxSequence.rollback()
}

internal class LexerIterator(givenSource: Iterator<Char>, initialSourceLocation: SourceLocation) : Iterator<Token> {

    private val source: TransactionalSequence<Pair<Char, SourceLocation>>
    init {
        source = IteratorBasedTransactionalSequence(
            SourceLocationAwareCharIterator(
                initialSourceLocation,
                givenSource
            )
        )
    }

    private var next: Token? = findNext()

    override fun hasNext() = next != null

    override fun next(): Token {
        if (next == null) throw NoSuchElementException()

        val token = next!!
        next = findNext()
        return token
    }

    /**
     * @return The nextToken token from the input or `null` if there are no more tokens.
     */
    private fun findNext(): Token? {
        skipWhitespace()

        if (!source.hasNext()) return null

        // try to match an operator
        val operatorToken = tryMatchOperator()
        if (operatorToken != null) return operatorToken

        if (!source.hasNext()) return null

        var text = collectUntilOperatorOrWhitespace()

        if (text.first[0].isDigit()) {
            // NUMERIC LITERAL
            val numberWithLocation: Pair<Number, SourceLocationRange>
            source.mark()

            var next = source.next()
            if (next.first == DECIMAL_SEPARATOR) {
                next = source.next()

                if (next.first.isDigit()) {
                    // <DIGIT...> <SEPARATOR> <DIGIT...> => floating point literal
                    source.commit()
                    val afterSeparator = collectUntilOperatorOrWhitespace()
                    val numberLocationRange = SourceLocationRange(text.second.start, afterSeparator.second)
                    val numberAsString = text.first + '.' + afterSeparator.first
                    numberWithLocation = Pair(numberAsString.toDouble(), numberLocationRange)
                }
                else {
                    // <DIGIT...> <SEPARATOR> <!DIGIT> => treat the separator as an operator; the nextCharsAsString invocation will
                    // find it and deal with it
                    source.rollback()
                    numberWithLocation = Pair(text.first.toLong(), text.second)
                }
            }
            else {
                numberWithLocation = Pair(text.first.toLong(), text.second)
                source.rollback()
            }

            return NumericLiteralToken(numberWithLocation.first, numberWithLocation.second)
        }
        else {
            // IDENTIFIER
            return IdentifierToken(text.first, text.second)
        }
    }

    private fun nextCharsAsString(n: Int): Pair<String, SourceLocationRange>? {
        if (n < 1) throw IllegalArgumentException()

        var start: SourceLocation? = null
        var end: SourceLocation? = null
        val charAr = CharArray(n)
        for (i in 0 until n) {
            if (source.hasNext()) {
                val next = source.next()
                charAr[i] = next.first

                if (start == null) {
                    start = next.second
                }

                end = next.second<commit_msg>Lexer: proper support for numeric literals<commit_after>package com.github.tmarsteel.ktprolog.parser.lexer

import com.github.tmarsteel.ktprolog.parser.sequence.IteratorBasedTransactionalSequence
import com.github.tmarsteel.ktprolog.parser.sequence.TransactionalSequence
import com.github.tmarsteel.ktprolog.parser.source.SourceLocation
import com.github.tmarsteel.ktprolog.parser.source.SourceLocationRange
import com.github.tmarsteel.ktprolog.parser.source.SourceUnit

class Lexer(source: Iterator<Char>, initialSourceLocation: SourceLocation) : TransactionalSequence<Token>
{
    private val lexerIteratorTxSequence = IteratorBasedTransactionalSequence(LexerIterator(source, initialSourceLocation))

    constructor(unit: SourceUnit, source: Iterator<Char>): this(source, SourceLocation(unit, 1, 0, 0))

    override fun mark() = lexerIteratorTxSequence.mark()

    override fun commit() = lexerIteratorTxSequence.commit()

    override fun hasNext() = lexerIteratorTxSequence.hasNext()

    override fun next() = lexerIteratorTxSequence.next()

    override fun rollback() = lexerIteratorTxSequence.rollback()
}

internal class LexerIterator(givenSource: Iterator<Char>, initialSourceLocation: SourceLocation) : Iterator<Token> {

    private val source: TransactionalSequence<Pair<Char, SourceLocation>>
    init {
        source = IteratorBasedTransactionalSequence(
            SourceLocationAwareCharIterator(
                initialSourceLocation,
                givenSource
            )
        )
    }

    private var next: Token? = findNext()

    override fun hasNext() = next != null

    override fun next(): Token {
        if (next == null) throw NoSuchElementException()

        val token = next!!
        next = findNext()
        return token
    }

    /**
     * @return The nextToken token from the input or `null` if there are no more tokens.
     */
    private fun findNext(): Token? {
        skipWhitespace()

        if (!source.hasNext()) return null

        // try to match an operator
        val operatorToken = tryMatchOperator()
        if (operatorToken != null) return operatorToken

        if (!source.hasNext()) return null

        var text = collectUntilOperatorOrWhitespace()

        if (text.first[0].isDigit()) {
            // NUMERIC LITERAL
            val numberWithLocation: Pair<Number, SourceLocationRange>
            source.mark()

            if (source.hasNext()) {
                var next = source.next()
                if (next.first == DECIMAL_SEPARATOR) {
                    source.mark()
                    next = source.next()

                    if (next.first.isDigit()) {
                        // <DIGIT...> <SEPARATOR> <DIGIT...> => floating point literal
                        source.rollback()
                        val afterSeparator = collectUntilOperatorOrWhitespace()
                        val numberLocationRange = SourceLocationRange(text.second.start, afterSeparator.second)
                        val numberAsString = text.first + '.' + afterSeparator.first
                        numberWithLocation = Pair(numberAsString.toDouble(), numberLocationRange)
                    } else {
                        // <DIGIT...> <SEPARATOR> <!DIGIT> => treat the separator as an operator; the nextCharsAsString invocation will
                        // find it and deal with it
                        source.commit()
                        source.rollback()
                        numberWithLocation = Pair(text.first.toLong(), text.second)
                    }
                } else {
                    numberWithLocation = Pair(text.first.toLong(), text.second)
                    source.rollback()
                }
            } else {
                numberWithLocation = Pair(text.first.toLong(), text.second)
                source.commit()
            }

            return NumericLiteralToken(numberWithLocation.first, numberWithLocation.second)
        }
        else {
            // IDENTIFIER
            return IdentifierToken(text.first, text.second)
        }
    }

    private fun nextCharsAsString(n: Int): Pair<String, SourceLocationRange>? {
        if (n < 1) throw IllegalArgumentException()

        var start: SourceLocation? = null
        var end: SourceLocation? = null
        val charAr = CharArray(n)
        for (i in 0 until n) {
            if (source.hasNext()) {
                val next = source.next()
                charAr[i] = next.first

                if (start == null) {
                    start = next.second
                }

                end = next.second
{'fraction_non_alphanumeric': 0.09121621621621621, 'fraction_numerical': 0.0013513513513513514, 'mean_word_length': 2.1106830122591944, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1920746', 'n_tokens_mistral': 2359, 'n_tokens_neox': 2200, 'n_words': 531}