<commit_before>package org.spoofax.jsglr2.tokens;

import static org.spoofax.jsglr.client.imploder.IToken.TK_EOF;
import static org.spoofax.jsglr.client.imploder.IToken.TK_RESERVED;

import java.util.ArrayList;
import java.util.Iterator;

import org.metaborg.parsetable.productions.IProduction;
import org.spoofax.jsglr.client.imploder.IToken;
import org.spoofax.jsglr.client.imploder.Token;
import org.spoofax.jsglr2.parser.Position;

public class Tokens implements IParseTokens {

    private static final long serialVersionUID = 2054391299757162697L;

    private final String fileName;
    private final String input;

    private IToken startToken, endToken;

    private final ArrayList<IToken> tokens;

    public Tokens(String input, String fileName) {
        this.input = input;
        this.fileName = fileName;

        this.tokens = new ArrayList<>();
    }

    public IToken startToken() {
        return startToken;
    }

    public IToken endToken() {
        return endToken;
    }

    public void makeStartToken() {
        startToken = new Token(this, fileName, 0, 1, 1, 0, -1, TK_RESERVED);

        addToken(startToken);
    }

    public void makeEndToken(Position endPosition) {
        endToken = new Token(this, fileName, tokens.size(), endPosition.line, endPosition.column - 1,
            endPosition.offset, -1, TK_EOF);

        addToken(endToken);
    }

    public IToken makeToken(Position startPosition, Position endPosition, IProduction production) {
        int tokenKind;

        if(startPosition.equals(endPosition)) {
            tokenKind = IToken.TK_NO_TOKEN_KIND;
        } else if(production == null) {
            tokenKind = IToken.TK_STRING; // indicates a character/int terminal, e.g. 'x'
        } else if(production.isLayout()) {
            tokenKind = IToken.TK_LAYOUT;
        } else if(production.isLiteral()) {
            if(production.isOperator())
                tokenKind = IToken.TK_OPERATOR;
            else
                tokenKind = IToken.TK_KEYWORD;
        } else if(production.isLexical()) {
            if(production.isStringLiteral())
                tokenKind = IToken.TK_STRING;
            else if(production.isNumberLiteral())
                tokenKind = IToken.TK_NUMBER;
            else
                tokenKind = IToken.TK_IDENTIFIER;
        } else {
            throw new IllegalStateException("invalid production/token type");
        }

        IToken token = new Token(this, fileName, tokens.size(), startPosition.line, startPosition.column,
            startPosition.offset, endPosition.offset - 1, tokenKind);

        addToken(token);

        return token;
    }

    public int addToken(IToken token) {
        int index = tokens.size();

        tokens.add(token);

        return index;
    }

    @Override public Iterator<IToken> iterator() {<commit_msg>Make JSGLR2's Tokens.addToken return void<commit_after>package org.spoofax.jsglr2.tokens;

import static org.spoofax.jsglr.client.imploder.IToken.TK_EOF;
import static org.spoofax.jsglr.client.imploder.IToken.TK_RESERVED;

import java.util.ArrayList;
import java.util.Iterator;

import javax.annotation.Nonnull;

import org.metaborg.parsetable.productions.IProduction;
import org.spoofax.jsglr.client.imploder.IToken;
import org.spoofax.jsglr.client.imploder.Token;
import org.spoofax.jsglr2.parser.Position;

public class Tokens implements IParseTokens {

    private static final long serialVersionUID = 2054391299757162697L;

    private final String fileName;
    private final String input;

    private IToken startToken, endToken;

    private final ArrayList<IToken> tokens;

    public Tokens(String input, String fileName) {
        this.input = input;
        this.fileName = fileName;

        this.tokens = new ArrayList<>();
    }

    public IToken startToken() {
        return startToken;
    }

    public IToken endToken() {
        return endToken;
    }

    public void makeStartToken() {
        startToken = new Token(this, fileName, 0, 1, 1, 0, -1, TK_RESERVED);

        addToken(startToken);
    }

    public void makeEndToken(Position endPosition) {
        endToken = new Token(this, fileName, tokens.size(), endPosition.line, endPosition.column - 1,
            endPosition.offset, -1, TK_EOF);

        addToken(endToken);
    }

    public IToken makeToken(Position startPosition, Position endPosition, IProduction production) {
        int tokenKind;

        if(startPosition.equals(endPosition)) {
            tokenKind = IToken.TK_NO_TOKEN_KIND;
        } else if(production == null) {
            tokenKind = IToken.TK_STRING; // indicates a character/int terminal, e.g. 'x'
        } else if(production.isLayout()) {
            tokenKind = IToken.TK_LAYOUT;
        } else if(production.isLiteral()) {
            if(production.isOperator())
                tokenKind = IToken.TK_OPERATOR;
            else
                tokenKind = IToken.TK_KEYWORD;
        } else if(production.isLexical()) {
            if(production.isStringLiteral())
                tokenKind = IToken.TK_STRING;
            else if(production.isNumberLiteral())
                tokenKind = IToken.TK_NUMBER;
            else
                tokenKind = IToken.TK_IDENTIFIER;
        } else {
            throw new IllegalStateException("invalid production/token type");
        }

        IToken token = new Token(this, fileName, tokens.size(), startPosition.line, startPosition.column,
            startPosition.offset, endPosition.offset - 1, tokenKind);

        addToken(token);

        return token;
    }

    private void addToken(IToken token) {
        tokens.add(token);
    }

    @Override @Nonnull public Iterator<IToken> iterator() {
{'fraction_non_alphanumeric': 0.09482155353393981, 'fraction_numerical': 0.010321903428971308, 'mean_word_length': 2.5753595997498437, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6542328', 'n_tokens_mistral': 1747, 'n_tokens_neox': 1635, 'n_words': 366}