<commit_before>package com.haskforce;

import com.haskforce.highlighting.HaskellSyntaxHighlightingLexer;
import com.haskforce.parsing.HaskellParser2;
import com.intellij.lang.ASTNode;
import com.intellij.lang.Language;
import com.intellij.lang.ParserDefinition;
import com.intellij.lang.PsiParser;
import com.intellij.lexer.Lexer;
import com.intellij.openapi.project.Project;
import com.intellij.psi.FileViewProvider;
import com.intellij.psi.PsiElement;
import com.intellij.psi.PsiFile;
import com.intellij.psi.TokenType;
import com.intellij.psi.tree.IFileElementType;
import com.intellij.psi.tree.TokenSet;
import org.jetbrains.annotations.NotNull;


import com.haskforce.parser.HaskellParser;
import com.haskforce.psi.HaskellFile;
import com.haskforce.psi.HaskellTypes;

public class HaskellParserDefinition implements ParserDefinition {
    public static final TokenSet WHITE_SPACES = TokenSet.create(TokenType.WHITE_SPACE);
    public static final TokenSet COMMENTS = TokenSet.create(HaskellTypes.COMMENT,
            HaskellTypes.NCOMMENT);
    public static final TokenSet STRINGS = TokenSet.create(HaskellTypes.STRINGTOKEN);

    public static final IFileElementType FILE = new IFileElementType(Language.<HaskellLanguage>findInstance(HaskellLanguage.class));

    @NotNull
    @Override
    public Lexer createLexer(Project project) {
        return new HaskellSyntaxHighlightingLexer();
    }

    @NotNull
    public TokenSet getWhitespaceTokens() {
        return WHITE_SPACES;
    }

    @NotNull
    public TokenSet getCommentTokens() {
        return COMMENTS;
    }

    @NotNull
    public TokenSet getStringLiteralElements() {
        return STRINGS;
    }

    @NotNull
    public PsiParser createParser(final Project project) {
        return new HaskellParser();
        // return new HaskellParser2(project);
    }

    @Override
    public IFileElementType getFileNodeType() {
        return FILE;
    }
<commit_msg>Added ability to switch to HaskellParser2 via VM arg -DPARSER=2<commit_after>package com.haskforce;

import com.haskforce.highlighting.HaskellSyntaxHighlightingLexer;
import com.haskforce.parsing.HaskellParser2;
import com.intellij.lang.ASTNode;
import com.intellij.lang.Language;
import com.intellij.lang.ParserDefinition;
import com.intellij.lang.PsiParser;
import com.intellij.lexer.Lexer;
import com.intellij.openapi.project.Project;
import com.intellij.psi.FileViewProvider;
import com.intellij.psi.PsiElement;
import com.intellij.psi.PsiFile;
import com.intellij.psi.TokenType;
import com.intellij.psi.tree.IFileElementType;
import com.intellij.psi.tree.TokenSet;
import org.jetbrains.annotations.NotNull;


import com.haskforce.parser.HaskellParser;
import com.haskforce.psi.HaskellFile;
import com.haskforce.psi.HaskellTypes;

public class HaskellParserDefinition implements ParserDefinition {
    public static final TokenSet WHITE_SPACES = TokenSet.create(TokenType.WHITE_SPACE);
    public static final TokenSet COMMENTS = TokenSet.create(HaskellTypes.COMMENT,
            HaskellTypes.NCOMMENT);
    public static final TokenSet STRINGS = TokenSet.create(HaskellTypes.STRINGTOKEN);

    public static final IFileElementType FILE = new IFileElementType(Language.<HaskellLanguage>findInstance(HaskellLanguage.class));

    @NotNull
    @Override
    public Lexer createLexer(Project project) {
        return new HaskellSyntaxHighlightingLexer();
    }

    @NotNull
    public TokenSet getWhitespaceTokens() {
        return WHITE_SPACES;
    }

    @NotNull
    public TokenSet getCommentTokens() {
        return COMMENTS;
    }

    @NotNull
    public TokenSet getStringLiteralElements() {
        return STRINGS;
    }

    @NotNull
    public PsiParser createParser(final Project project) {
        // Don't do System.getProperty("PARSER").equals("2"), or java.lang.NullPointerException!
        if ("2".equals(System.getProperty("PARSER"))) {
            return new HaskellParser2(project);
        }
        return new HaskellParser();
    }

    @Override
    public IFileElementType getFileNodeType() {
        return FILE;
    }

{'fraction_non_alphanumeric': 0.08439334637964775, 'fraction_numerical': 0.0019569471624266144, 'mean_word_length': 4.12406015037594, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7478956', 'n_tokens_mistral': 1279, 'n_tokens_neox': 1215, 'n_words': 244}