<commit_before>package ambroscum.parser;

import ambroscum.errors.SyntaxError;

public class Tokenizer
{
	public static TokenStream tokenize(String str)
	{
		TokenStream stream = new TokenStream();
		
		for (int i = 0; i < str.length(); i++)
		{
			if (str.charAt(i) == '\n')
			{
				stream.offer(Token.NEWLINE);
				while (i + 1 < str.length() && str.charAt(i + 1) == '\t')
				{
					i++;
					stream.offer(Token.TAB);
				}
				continue;
			}
			if (str.charAt(i) == ' ')
				throw new SyntaxError("Unexpected whitespace");
			if (str.charAt(i) == '"')
			{
				int j = i++;
				while (i < str.length() && str.charAt(i) != '\n' && str.charAt(i) != '"')
					i++;
				if (i == str.length() || str.charAt(i) == '\n')
					throw new SyntaxError("Nonterminating string");
				stream.offer(new Token(str.substring(j, i + 1)));
				if (i + 1 == str.length())
					break;
				if (str.charAt(i + 1) == '\n')
					continue;
				i++;
				if (str.charAt(i) != ' ')
					throw new SyntaxError("Missing whitespace");
				continue;
			}
			if (openParen(str.charAt(i)))
			{
				stream.offer(new Token("" + str.charAt(i)));
				continue;
			}
			if (closeParen(str.charAt(i)) || str.charAt(i) == ',' || str.charAt(i) == '.')
				throw new SyntaxError("Unexpected token: " + str.charAt(i));
			int j = i++;
			while (i < str.length() && !isSeparator(str.charAt(i)) && !isWhitespace(str.charAt(i)))
				i++;
			stream.offer(new Token(str.substring(j, i)));
			/*if (isSeparator(str.charAt(i)))
				stream.offer(getToken(str.charAt(i)));
			if (str.charAt(i) == '\t')
				throw new SyntaxError("Unexpected tab");
			while (i < str.length() && closeParen(str.charAt(i)))
			{
 				while (str.charAt(i + 1) == '\n')
					st
				i++;
				if (str.charAt(i) != ' ')
					throw new SyntaxError("Missing whitespace");
			}
			if (str.charAt(i) == ',')
			{
				i++;
				stream.offer(Token.COMMA);
			}*/
			while (i < str.length() && isSeparator(str.charAt(i)))
				stream.offer(getToken(str.charAt(i++)));
			if (i == str.length())
				break;
			if (!isWhitespace(str.charAt(i)))
				throw new SyntaxError("Whitespace expected");
			if (str.charAt(i) == '\t')
				throw new SyntaxError("Unexpected tab");
			if (str.charAt(i) == '\n')
				i--;
		}
		return stream;
	}
	
	private static Token getToken(char c)
	{
		switch (c)
		{
			case '\n': return Token.NEWLINE;
			case '.': return Token.DOT;
			case ',': return Token.COMMA;
		}
		return new Token("" + c);
	}
	
	private static boolean isSeparator(char c)
	{
		return openParen(c) || closeParen(c) || c == ',' || c == '.';
	}
	
	private static boolean isWhitespace(char c)
	{
		return c == ' ' || c == '\n' || c == '\t';
	}
	
	private static boolean openParen(char c)
	{
		return c == '(' || c == '[' || c == '{';
	}<commit_msg>Fixed tokenizer, I think, take 2<commit_after>package ambroscum.parser;

import ambroscum.errors.SyntaxError;

public class Tokenizer
{
	public static TokenStream tokenize(String str)
	{
		TokenStream stream = new TokenStream();
		
		for (int i = 0; i < str.length(); i++)
		{
			if (str.charAt(i) == '\n')
			{
				stream.offer(Token.NEWLINE);
				while (i + 1 < str.length() && str.charAt(i + 1) == '\t')
				{
					i++;
					stream.offer(Token.TAB);
				}
				continue;
			}
			if (str.charAt(i) == ' ')
				throw new SyntaxError("Unexpected whitespace");
			if (str.charAt(i) == '"')
			{
				int j = i++;
				while (i < str.length() && str.charAt(i) != '\n' && str.charAt(i) != '"')
					i++;
				if (i == str.length() || str.charAt(i) == '\n')
					throw new SyntaxError("Nonterminating string");
				stream.offer(new Token(str.substring(j, i + 1)));
				if (i + 1 == str.length())
					break;
				if (str.charAt(i + 1) == '\n')
					continue;
				i++;
				if (str.charAt(i) != ' ')
					throw new SyntaxError("Missing whitespace");
				continue;
			}
			if (openParen(str.charAt(i)))
			{
				stream.offer(new Token("" + str.charAt(i)));
				continue;
			}
			if (closeParen(str.charAt(i)) || str.charAt(i) == ',' || str.charAt(i) == '.')
				throw new SyntaxError("Unexpected token: " + str.charAt(i));
			int j = i++;
			while (i < str.length() && !isSeparator(str.charAt(i)) && !isWhitespace(str.charAt(i)))
				i++;
			stream.offer(new Token(str.substring(j, i)));
			while (i < str.length() && closeParen(str.charAt(i)))
				stream.offer(getToken(str.charAt(i++)));
			if (i == str.length())
				break;
			if (str.charAt(i) == ',')
			{
				stream.offer(Token.COMMA);
				i++;
			}
			if (!isWhitespace(str.charAt(i)) && !openParen(str.charAt(i)))
				throw new SyntaxError("Whitespace expected: " + i);
			if (str.charAt(i) == '\t')
				throw new SyntaxError("Unexpected tab");
			if (str.charAt(i) == '\n' || openParen(str.charAt(i)))
				i--;
		}
		return stream;
	}
	
	private static Token getToken(char c)
	{
		switch (c)
		{
			case '\n': return Token.NEWLINE;
			case '.': return Token.DOT;
			case ',': return Token.COMMA;
		}
		return new Token("" + c);
	}
	
	private static boolean isSeparator(char c)
	{
		return openParen(c) || closeParen(c) || c == ',' || c == '.';
	}
	
	private static boolean isWhitespace(char c)
	{
		return c == ' ' || c == '\n' || c == '\t';
	}
	
	private static boolean openParen(char c)
	{
		return c == '(' || c == '[' || c == '{';
	}
{'fraction_non_alphanumeric': 0.21021536115875739, 'fraction_numerical': 0.0026681913474366306, 'mean_word_length': 2.8503301540719, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5733414', 'n_tokens_mistral': 2406, 'n_tokens_neox': 1992, 'n_words': 426}