<commit_before>/*
InfixTokenizer.java

Licensed under MIT/X11-License. License text can be found in LICENSE file in root folder.

*/
package infix;

import java.util.ArrayList;
import java.util.Enumeration;

public class InfixTokenizer implements Enumeration<String> {
	
	private ArrayList<String> 	tokens 	= new ArrayList<String>();
	private String 				buffer 	= "";
	private int					pointer	= 0;
	
	public InfixTokenizer(String term){
		// split term on "()*/+-^"
		char[] chars = term.toCharArray();
		for(int i = 0; i < chars.length; i++){
			switch (chars[i]) {
			case '(':
				tokenFiller();
				tokens.add("(");
				break;
			case ')':
				tokenFiller();
				tokens.add(")");
				break;
			case '*':
				tokenFiller();
				tokens.add("*");
				break;
			case '/':
				tokenFiller();
				tokens.add("/");
				break;
			case '+':
				tokenFiller();
				tokens.add("+");
				break;
			case '-':
				if(isOperator(chars[i-1])){
					buffer += chars[i];
				} else {
					tokenFiller();
					tokens.add("-");
				}
				break;
			case '^':
				tokenFiller();
				tokens.add("^");
				break;
			default:
				buffer += chars[i];
				break;
			}
		}
	}
	
	private boolean isOperator(char c){
		if (c == '+' || c == '-' || c == '*' || c == '/' || c == '(' || c == '^') return true;
		return false;
	}
	
	private void tokenFiller(){
		if (buffer.length() > 0) tokens.add(buffer);
		buffer = "";
	}

	@Override
	public boolean hasMoreElements() {
		if (pointer < tokens.size()) return true;
		return false;
	}
	
	public boolean hasMoreTokens() {
		// kept for compatibility to StringTokenizer
		return hasMoreElements();
	}

	@Override
	public String nextElement() {
		String element = tokens.get(pointer);
		pointer++;
		return element;
	}
	
	public String nextToken() {
		// kept for compatibility to StringTokenizer
		return nextElement();
	}

}
<commit_msg>Fixes #1. Implemented right-associativity for exponents<commit_after>/*
InfixTokenizer.java

Licensed under MIT/X11-License. License text can be found in LICENSE file in root folder.

*/
package infix;

import java.util.ArrayList;
import java.util.Enumeration;

public class InfixTokenizer implements Enumeration<String> {
	
	private ArrayList<String> 	tokens 	= new ArrayList<String>();
	private String 				buffer 	= "";
	private int					pointer	= 0;
	
	public InfixTokenizer(String term){
		// to implement right-associativity exponents needs to be in brackets
		for(int i = 0; i < term.length(); i++){
			String character = term.substring(i,i+1);
			if(character.equals("^")){
				term = term.substring(0, i+1) + "(" + term.substring(i+1) + ")";
				i+=2;
			}
		}
		// split term on "()*/+-^"
		char[] chars = term.toCharArray();
		for(int i = 0; i < chars.length; i++){
			switch (chars[i]) {
			case '(':
				tokenFiller();
				tokens.add("(");
				break;
			case ')':
				tokenFiller();
				tokens.add(")");
				break;
			case '*':
				tokenFiller();
				tokens.add("*");
				break;
			case '/':
				tokenFiller();
				tokens.add("/");
				break;
			case '+':
				tokenFiller();
				tokens.add("+");
				break;
			case '-':
				if(isOperator(chars[i-1])){
					buffer += chars[i];
				} else {
					tokenFiller();
					tokens.add("-");
				}
				break;
			case '^':
				tokenFiller();
				tokens.add("^");
				break;
			default:
				buffer += chars[i];
				break;
			}
		}
	}
	
	private boolean isOperator(char c){
		if (c == '+' || c == '-' || c == '*' || c == '/' || c == '(' || c == '^') return true;
		return false;
	}
	
	private void tokenFiller(){
		if (buffer.length() > 0) tokens.add(buffer);
		buffer = "";
	}

	@Override
	public boolean hasMoreElements() {
		if (pointer < tokens.size()) return true;
		return false;
	}
	
	public boolean hasMoreTokens() {
		// kept for compatibility to StringTokenizer
		return hasMoreElements();
	}

	@Override
	public String nextElement() {
		String element = tokens.get(pointer);
		pointer++;
		return element;
	}
	
	public String nextToken() {
		// kept for compatibility to StringTokenizer
		return nextElement();
	}

}

{'fraction_non_alphanumeric': 0.1708405653359782, 'fraction_numerical': 0.004711133151500124, 'mean_word_length': 2.8900675024108002, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1748324', 'n_tokens_mistral': 1668, 'n_tokens_neox': 1347, 'n_words': 341}