<commit_before>package utilities;

import java.util.ArrayList;

public class TokenParser {
	
	public TokenParser() {}
	
	public ArrayList<String> parse(String line) {
		ArrayList<String> tokens = new ArrayList<String>();
		line += " ";
		while( line.contains("  ") ) {
			line.replace("  ", " ");
		}
		char[] charArray = line.toCharArray();
		String temp = "";
		boolean openQuote = false;
		for( char c : charArray ) {
			if( c == ' ' && !openQuote) {
				tokens.add(temp);
				temp = "";
			} else if( c == '"' ) {
				if( openQuote ) {
					openQuote = false;
					tokens.add(temp);
					temp = "";
				} else {
					openQuote = true;
				}
			} else {
				temp += c;
			}
		}
		return tokens;
	}
	
}<commit_msg>updated token parsing to handle quotees and spaces better<commit_after>package utilities;

import java.util.ArrayList;

public class TokenParser {
	
	public TokenParser() {}
	
	public ArrayList<String> parse(String line) {
		ArrayList<String> tokens = new ArrayList<String>();
		line += " ";
		while( line.contains("  ") ) {
			line.replace("  ", " ");
		}
		char[] charArray = line.toCharArray();
		String temp = "";
		boolean openQuote = false;
		boolean border = false;
		for( char c : charArray ) {
			if( c == ' ' && !openQuote && temp.length() > 0 ) {
				tokens.add(temp);
				temp = "";
				border = false;
			} else if( c == '"' ) {
				if( openQuote ) {
					openQuote = false;
					if( temp.length() > 0 ) {
						tokens.add(temp);
					}
					temp = "";
					border = true;
				} else {
					openQuote = true;
				}
			} else {
				if( !border ) {
					temp += c;
				}
				border = false;
			}
		}
		return tokens;
	}
	
}
{'fraction_non_alphanumeric': 0.1618181818181818, 'fraction_numerical': 0.0012121212121212121, 'mean_word_length': 2.439583333333333, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5875951', 'n_tokens_mistral': 706, 'n_tokens_neox': 544, 'n_words': 134}