<commit_before>package bschecker;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;

import bschecker.bluesheets.Bluesheet;
import bschecker.bluesheets.Bluesheets;
import bschecker.bluesheets.QuotationForm;
import bschecker.util.ErrorList;
import bschecker.util.Tools;
import bschecker.util.UtilityMethods;
import opennlp.tools.cmdline.parser.ParserTool;
import opennlp.tools.cmdline.postag.POSModelLoader;
import opennlp.tools.namefind.NameFinderME;
import opennlp.tools.namefind.TokenNameFinderModel;
import opennlp.tools.parser.AbstractBottomUpParser;
import opennlp.tools.parser.Parse;
import opennlp.tools.parser.Parser;
import opennlp.tools.parser.ParserFactory;
import opennlp.tools.parser.ParserModel;
import opennlp.tools.postag.POSModel;
import opennlp.tools.postag.POSSample;
import opennlp.tools.postag.POSTaggerME;
import opennlp.tools.sentdetect.SentenceDetectorME;
import opennlp.tools.sentdetect.SentenceModel;
import opennlp.tools.tokenize.Tokenizer;
import opennlp.tools.tokenize.TokenizerME;
import opennlp.tools.tokenize.TokenizerModel;
import opennlp.tools.util.InvalidFormatException;
import opennlp.tools.util.Span;

/**
 * this class contains examples of how to use openNLP and can be used for testing purposes
 * to use enter your input text in the literal for the input String
 * then uncomment which ever tools you would like to run
 * 
 * this class can also be used to test bluesheet classes without launching the application
 * to do this first uncomment the lower portion of the main method
 * if the value in the boolean array at index n is true, the (n + 1)th bluesheet will be tested for
 * 
 * @author tedpyne
 * @author JeremiahDeGreeff
 */
@SuppressWarnings("unused")
public class BSChecker {
	
	public static void main(String[] args) throws InvalidFormatException, IOException{
		String input = "";
		input = UtilityMethods.replaceInvalidChars(input);
		input = UtilityMethods.removeExtraPunctuation(input, 0, new ArrayList<Integer>());
		input += "\n";
		
//		Tokenize(input);
//		SentenceDetect(input);
//		POStag(input);
//		findNames(input);
//		parse(input);
		
//		Main.initialize();
//		Bluesheets.loadSettings(new boolean[]{false, false, false, false, false, false, false, false, false, false, false, false, false, false});
//		System.out.println("input: " + input);
//		ErrorList errors = Bluesheet.findAllErrors(input);
	}
	
	
	public static void Tokenize(String input) throws InvalidFormatException, IOException {
		InputStream is = new FileInputStream("lib/en-token.bin");
		TokenizerModel model = new TokenizerModel(is);
		is.close();
		Tokenizer tokenizer = new TokenizerME(model);
		
		String[] tokens = tokenizer.tokenize(input);<commit_msg>minor tweak<commit_after>package bschecker;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;

import bschecker.bluesheets.Bluesheet;
import bschecker.bluesheets.Bluesheets;
import bschecker.bluesheets.QuotationForm;
import bschecker.util.ErrorList;
import bschecker.util.Tools;
import bschecker.util.UtilityMethods;
import opennlp.tools.cmdline.parser.ParserTool;
import opennlp.tools.cmdline.postag.POSModelLoader;
import opennlp.tools.namefind.NameFinderME;
import opennlp.tools.namefind.TokenNameFinderModel;
import opennlp.tools.parser.AbstractBottomUpParser;
import opennlp.tools.parser.Parse;
import opennlp.tools.parser.Parser;
import opennlp.tools.parser.ParserFactory;
import opennlp.tools.parser.ParserModel;
import opennlp.tools.postag.POSModel;
import opennlp.tools.postag.POSSample;
import opennlp.tools.postag.POSTaggerME;
import opennlp.tools.sentdetect.SentenceDetectorME;
import opennlp.tools.sentdetect.SentenceModel;
import opennlp.tools.tokenize.Tokenizer;
import opennlp.tools.tokenize.TokenizerME;
import opennlp.tools.tokenize.TokenizerModel;
import opennlp.tools.util.InvalidFormatException;
import opennlp.tools.util.Span;

/**
 * this class contains examples of how to use openNLP and can be used for testing purposes
 * to use enter your input text in the literal for the input String
 * then uncomment which ever tools you would like to run
 * 
 * this class can also be used to test bluesheet classes without launching the application
 * to do this first uncomment the lower portion of the main method
 * if the value in the boolean array at index n is true, the (n + 1)th bluesheet will be tested for
 * 
 * @author tedpyne
 * @author JeremiahDeGreeff
 */
@SuppressWarnings("unused")
public class BSChecker {
	
	public static void main(String[] args) throws InvalidFormatException, IOException{
		String input = "";
		input = UtilityMethods.replaceInvalidChars(input);
		input = UtilityMethods.removeExtraPunctuation(input, 0, new ArrayList<Integer>());
		input += "\n";
		
//		Tokenize(input);
//		SentenceDetect(input);
//		POStag(input);
//		findNames(input);
//		parse(input);
		
//		Main.initialize();
//		Bluesheets.loadSettings(new boolean[]{false, false, false, false, false, false, false, false, false, false, false, false, false, false});
//		System.out.print("\ninput: " + input);
//		ErrorList errors = Bluesheet.findAllErrors(input);
	}
	
	
	public static void Tokenize(String input) throws InvalidFormatException, IOException {
		InputStream is = new FileInputStream("lib/en-token.bin");
		TokenizerModel model = new TokenizerModel(is);
		is.close();
		Tokenizer tokenizer = new TokenizerME(model);
		
		String[] tokens = tokenizer.tokenize(input);
{'fraction_non_alphanumeric': 0.10046728971962617, 'fraction_numerical': 0.0007189072609633358, 'mean_word_length': 5.374570446735395, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5986711', 'n_tokens_mistral': 1775, 'n_tokens_neox': 1696, 'n_words': 448}