<commit_before>package org.jetbrains.haskell.parser.lexer

import com.intellij.lexer.LexerBase
import com.intellij.psi.tree.IElementType
import com.intellij.psi.TokenType
import org.jetbrains.haskell.highlight.HaskellHighlighter
import org.jetbrains.haskell.parser.token.*
import org.jetbrains.haskell.parser.HaskellToken


/**
 * Created by atsky on 3/6/14.
 */

public class HaskellLexer() : LexerBase() {
    class object {
        val COMMENT_RULE: State<Boolean> = str("{-") + merge(noneOf("-"), str("-") + noneOf("}")).star() + str("-}")
        val fsm: State<IElementType> = makeFsm();

        fun makeFsm(): State<IElementType> {
            val fsm = buildFsm<IElementType> {

                add(TokenType.WHITE_SPACE,
                        oneOf(" \t").plus())

                add(TokenType.NEW_LINE_INDENT,
                        str("\n") + oneOf(" \t").star())

                add(END_OF_LINE_COMMENT,
                        str("--") + noneOf("\n").star())

                add(PRAGMA,
                        str("#") + noneOf("\n").star())


                add(COMMENT,
                        COMMENT_RULE)

                for (keyword in KEYWORDS) {
                    add(keyword, str(keyword.myName))
                }

                for (operator in OPERATORS) {
                    add(operator, str(operator.myName))
                }

                add(OPERATOR,
                        oneOf("!#$%&*+./<=>?@\\^|-~:").plus())

                add(ID,
                        oneOf('a'..'z') + oneOf(('a'..'z') + ('A'..'Z') + ('0'..'9') + '_' + '\'').star())

                add(TYPE_OR_CONS,
                        oneOf('A'..'Z') + oneOf(('a'..'z') + ('A'..'Z') + ('0'..'9') + '_' + '\'').star())

                val decimal = oneOf('0'..'9').plus()
                val octalNumber = str("0") + oneOf("oO") + oneOf('0'..'7').plus()


                add(NUMBER,
                        merge(decimal, octalNumber))

                add(STRING,
                        str("\"") + merge(noneOf("\""), str("\\\"")).star() + oneOf("\"\n"))

                add(CHARACTER,
                        str("'") + merge(noneOf("'"), str("\\'")).star() + str("'"))
<commit_msg>Fix lexer for strings<commit_after>package org.jetbrains.haskell.parser.lexer

import com.intellij.lexer.LexerBase
import com.intellij.psi.tree.IElementType
import com.intellij.psi.TokenType
import org.jetbrains.haskell.highlight.HaskellHighlighter
import org.jetbrains.haskell.parser.token.*
import org.jetbrains.haskell.parser.HaskellToken


/**
 * Created by atsky on 3/6/14.
 */

public class HaskellLexer() : LexerBase() {
    class object {
        val COMMENT_RULE: State<Boolean> = str("{-") + merge(noneOf("-"), str("-") + noneOf("}")).star() + str("-}")
        val fsm: State<IElementType> = makeFsm();

        fun makeFsm(): State<IElementType> {
            val fsm = buildFsm<IElementType> {

                add(TokenType.WHITE_SPACE,
                        oneOf(" \t").plus())

                add(TokenType.NEW_LINE_INDENT,
                        str("\n") + oneOf(" \t").star())

                add(END_OF_LINE_COMMENT,
                        str("--") + noneOf("\n").star())

                add(PRAGMA,
                        str("#") + noneOf("\n").star())


                add(COMMENT,
                        COMMENT_RULE)

                for (keyword in KEYWORDS) {
                    add(keyword, str(keyword.myName))
                }

                for (operator in OPERATORS) {
                    add(operator, str(operator.myName))
                }

                add(OPERATOR,
                        oneOf("!#$%&*+./<=>?@\\^|-~:").plus())

                add(ID,
                        oneOf('a'..'z') + oneOf(('a'..'z') + ('A'..'Z') + ('0'..'9') + '_' + '\'').star())

                add(TYPE_OR_CONS,
                        oneOf('A'..'Z') + oneOf(('a'..'z') + ('A'..'Z') + ('0'..'9') + '_' + '\'').star())

                val decimal = oneOf('0'..'9').plus()
                val octalNumber = str("0") + oneOf("oO") + oneOf('0'..'7').plus()


                add(NUMBER,
                        merge(decimal, octalNumber))


                val strSymbols = "abfnrtvx\\\"'"

                add(STRING,
                        str("\"") + merge(noneOf("\"\\"), str("\\") + oneOf(strSymbols)).star() + oneOf("\"\n"))

                add(CHARACTER,
                        str("'") + merge(noneOf("'\\"), str("\\") + oneOf(strSymbols)).star() + str("'"))

{'fraction_non_alphanumeric': 0.19623893805309733, 'fraction_numerical': 0.005752212389380531, 'mean_word_length': 1.5172605790645879, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1790658', 'n_tokens_mistral': 1529, 'n_tokens_neox': 1401, 'n_words': 218}