<commit_before>package com.github.ktrnka.droidling;

import java.util.ArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class Tokenizer
	{
	public static final Pattern wordPattern = Pattern.compile("\\s+");
	public static final Pattern frontMatter = Pattern.compile("([^\\w@]+)([\\w@].*)");
	public static final Pattern backMatter = Pattern.compile("(.*\\w)(\\W+)");
	public static final Pattern endsInAbbreviation = Pattern.compile(".*(Mr|Mrs|Dr|Jr|Ms|Prof|Sr|dept|Univ|Inc|Ltd|Co|Corp|Mt|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Oct|Nov|Dec|Sept|vs|etc|no)\\.");
	public static final Pattern sentencePattern = Pattern.compile("(?<=[\\.?!][^\\w\\s]?)\\s+(?![a-z])");
	public static final Pattern nonwordPattern = Pattern.compile("[^\\w']|<s>|</s>");
	public static final Pattern nospacePunctPattern = Pattern.compile("[,\\.?!:;]");
	
	public static final String messageStart = "<s>";
	public static final String messageEnd = "</s>";
	
	public static boolean isNonword(String word)
		{
		// This is the regular expression version of the code (and Java regex is slow)
		//return nonwordPattern.matcher(word).find();
		
		for (int i = 0; i < word.length(); i++)
			{
			char c = word.charAt(i);
			if (!Character.isLetter(c) && c != '\'')
				{
				return true;
				}
			}
		
		return false;
		}
	
	/**
	 * Regular expression-based anything in Java is slow as shit so I had to write this.
	 * I haven't tested it thoroughly enough yet, seems to help in a minor way.
	 * @param text
	 * @param delim<commit_msg>Updated comment.<commit_after>package com.github.ktrnka.droidling;

import java.util.ArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class Tokenizer
	{
	public static final Pattern wordPattern = Pattern.compile("\\s+");
	public static final Pattern frontMatter = Pattern.compile("([^\\w@]+)([\\w@].*)");
	public static final Pattern backMatter = Pattern.compile("(.*\\w)(\\W+)");
	public static final Pattern endsInAbbreviation = Pattern.compile(".*(Mr|Mrs|Dr|Jr|Ms|Prof|Sr|dept|Univ|Inc|Ltd|Co|Corp|Mt|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Oct|Nov|Dec|Sept|vs|etc|no)\\.");
	public static final Pattern sentencePattern = Pattern.compile("(?<=[\\.?!][^\\w\\s]?)\\s+(?![a-z])");
	public static final Pattern nonwordPattern = Pattern.compile("[^\\w']|<s>|</s>");
	public static final Pattern nospacePunctPattern = Pattern.compile("[,\\.?!:;]");
	
	public static final String messageStart = "<s>";
	public static final String messageEnd = "</s>";
	
	/**
	 * Check if a word isn't appropriate to list in the various displays.  Examples
	 * include numeric tokens and such.
	 * 
	 * TODO: This isn't appropriate for many languages that have hyphenated words.
	 * It's also inappropriate to filter out a trigram like "3rd and Seneca"
	 */
	public static boolean isNonword(String word)
		{
		// This is the regular expression version of the code (and Java regex is slow)
		//return nonwordPattern.matcher(word).find();
		
		for (int i = 0; i < word.length(); i++)
			{
			char c = word.charAt(i);
			if (!Character.isLetter(c) && c != '\'')
				{
				return true;
				}
			}
		
		return false;
		}
	
	/**
	 * Regular expression-based anything in Java is slow as shit so I had to write this.
	 * I haven't tested it thoroughly enough yet, seems to help in a minor way.
	 * @param text
	 * @param delim
{'fraction_non_alphanumeric': 0.1657722122838402, 'fraction_numerical': 0.0008944543828264759, 'mean_word_length': 4.0299850074962515, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 2, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4432879', 'n_tokens_mistral': 1179, 'n_tokens_neox': 1058, 'n_words': 313}