<commit_before>package footran.lexer;

import static footran.lexer.TokenType.*;

import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

import footran.reader.Card;

class LexerImpl extends StatementHandler {

	private String stmt;
	private int offset;
	private char c;
	/**
	 * activated when STOP was read to indicate, that the following CONST_INT
	 * will be read as octal
	 */
	private boolean octalMode = false;

	/**
	 * Creates a new Lexer based on the given stream of {@link Card}s.
	 * 
	 * @param cards
	 *            to lex
	 */
	public LexerImpl(Iterable<Card> cards) {
		this(cards.iterator());
	}

	/**
	 * Creates a new Lexer based on the given {@link Card}s.
	 * 
	 * @param cards
	 *            to lex
	 */
	public LexerImpl(Iterator<Card> cards) {
		super(cards);
	}

	@Override
	protected List<Token> lex(String statement) {
		List<Token> tokens = new ArrayList<>();
		offset = -1;
		c = 0;
		stmt = statement;
		nextChar();

		Token l = start();
		// TODO better control flow
		while (l != null) {
			tokens.add(l);
			l = start();
		}

		return tokens;
	}

	private Token start() {
		while (c != 0 && Character.isWhitespace(c))
			nextChar();
		if (c == 0)
			return null;

		int start = offset;
		Token t = null;
		
		if (Character.isLetter(c))
			t = ident();
		else if (Character.isDigit(c) || c == '.')
			t = constant();
		else
			t = misc();
		
		if (t == null)
			throw new LexicalException("invalid token at " + lineNo + ":"
					+ start + " \"" + stmt.substring(start, offset) + "\"");
		
		// activate octal mode on STOP/PAUSE, deactivate otherwise
		octalMode = t.getType() == STOP || t.getType() == PAUSE;
		
		return t;
	}

	private Token ident() {
		int start = offset;
		while (Character.isLetterOrDigit(c))
			nextChar();
		String text = stmt.substring(start, offset);
		TokenType type = getIdentType(text);

		return createToken(type, start, offset);
	}

	private TokenType getIdentType(String ident) {
		if (Keywords.isKeyword(ident))
			// a reserved keyword
			return TokenType.valueOf(ident);
		else {
			// some identifier
			int start = offset - ident.length();
			/**
			 * Functions.
			 * 
			 * The name of the function is 4 to 7 alphabetic or numeric
			 * characters (not special characters), of which the last must be F
			 * and the first must be alphabetic. Also, the first must be X if
			 * and only if the value of the function is to be fixed point.
			 */
			// "if a subscripted variable has 4 or more characters in its name, the last of these must not be an F"
			if (ident.length() >= 4 && ident.endsWith("F")) {
				if (ident.length() > 7)
					warn("function identifier too long (up to 7 characters allowed)", start);
				
				if (ident.startsWith("X"))
					return FUNC_INT;
				else
					return FUNC_FLOAT;
			} else {
				/**
				 * Floating Point Variables.
				 * 
				 * 1 to 6 alphabetic or numeric characters (not special characters)
				 * of which the first is alphabetic but not I, J, K, L, M, or N.
				 */
				/**
				 * Fixed Point Variables.
				 * 
				 * 1 to 6 alphabetic or numeric characters (not special characters)
				 * of which the first is I, J, K, l, M, or N.
				 */
				if (ident.length() > 6)
					warn("variable identifier too long (up to 6 characters allowed)", start);
				if (ident.charAt(0) >= 'I' && ident.charAt(0) <= 'N')
					return VAR_INT;<commit_msg>some TODOs for the lexer<commit_after>package footran.lexer;

import static footran.lexer.TokenType.*;

import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

import footran.reader.Card;

class LexerImpl extends StatementHandler {

	private String stmt;
	private int offset;
	private char c;
	/**
	 * activated when STOP was read to indicate, that the following CONST_INT
	 * will be read as octal
	 */
	private boolean octalMode = false;
	// TODO FORMAT seems to work different

	/**
	 * Creates a new Lexer based on the given stream of {@link Card}s.
	 * 
	 * @param cards
	 *            to lex
	 */
	public LexerImpl(Iterable<Card> cards) {
		this(cards.iterator());
	}

	/**
	 * Creates a new Lexer based on the given {@link Card}s.
	 * 
	 * @param cards
	 *            to lex
	 */
	public LexerImpl(Iterator<Card> cards) {
		super(cards);
	}

	@Override
	protected List<Token> lex(String statement) {
		List<Token> tokens = new ArrayList<>();
		offset = -1;
		c = 0;
		stmt = statement;
		nextChar();

		Token l = start();
		// TODO better control flow
		while (l != null) {
			tokens.add(l);
			l = start();
		}

		return tokens;
	}

	private Token start() {
		while (c != 0 && Character.isWhitespace(c))
			nextChar();
		if (c == 0)
			return null;

		int start = offset;
		Token t = null;
		
		if (Character.isLetter(c))
			t = ident();
		else if (Character.isDigit(c) || c == '.')
			t = constant();
		else
			t = misc();
		
		if (t == null)
			throw new LexicalException("invalid token at " + lineNo + ":"
					+ start + " \"" + stmt.substring(start, offset) + "\"");
		
		// activate octal mode on STOP/PAUSE, deactivate otherwise
		octalMode = t.getType() == STOP || t.getType() == PAUSE;
		
		return t;
	}

	private Token ident() {
		int start = offset;
		while (Character.isLetterOrDigit(c))
			nextChar();
		String text = stmt.substring(start, offset);
		TokenType type = getIdentType(text);

		return createToken(type, start, offset);
	}

	private TokenType getIdentType(String ident) {
		if (Keywords.isKeyword(ident))
			// a reserved keyword
			return TokenType.valueOf(ident);
		else {
			// some identifier
			int start = offset - ident.length();
			/**
			 * Functions.
			 * 
			 * The name of the function is 4 to 7 alphabetic or numeric
			 * characters (not special characters), of which the last must be F
			 * and the first must be alphabetic. Also, the first must be X if
			 * and only if the value of the function is to be fixed point.
			 */
			// "if a subscripted variable has 4 or more characters in its name, the last of these must not be an F"
			// TODO allow for non-subscripted variables >4 chars to end on F
			if (ident.length() >= 4 && ident.endsWith("F")) {
				if (ident.length() > 7)
					warn("function identifier too long (up to 7 characters allowed)", start);
				
				if (ident.startsWith("X"))
					return FUNC_INT;
				else
					return FUNC_FLOAT;
			} else {
				/**
				 * Floating Point Variables.
				 * 
				 * 1 to 6 alphabetic or numeric characters (not special characters)
				 * of which the first is alphabetic but not I, J, K, L, M, or N.
				 */
				/**
				 * Fixed Point Variables.
				 * 
				 * 1 to 6 alphabetic or numeric characters (not special characters)
				 * of which the first is I, J, K, l, M, or N.
				 */
				if (ident.length() > 6)
					warn("variable identifier too long (up to 6 characters allowed)", start);
				if (ident.charAt(0) >= 'I' && ident.charAt(0) <= 'N')
					return VAR_INT;
{'fraction_non_alphanumeric': 0.12250836607012949, 'fraction_numerical': 0.005383384257238469, 'mean_word_length': 2.7297883884970155, 'pattern_counts': {'":': 2, '<': 15, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7308034', 'n_tokens_mistral': 2622, 'n_tokens_neox': 2136, 'n_words': 780}