package edu.buffalo.cse.ir.wikiindexer.tokenizer.rules;

import java.util.Arrays;
import java.util.HashSet;

import edu.buffalo.cse.ir.wikiindexer.tokenizer.TokenStream;
import edu.buffalo.cse.ir.wikiindexer.tokenizer.TokenizerException;
import edu.buffalo.cse.ir.wikiindexer.tokenizer.rules.TokenizerRule.RULENAMES;

@RuleClass(className = RULENAMES.STOPWORDS)
public class StopwordsDefault implements TokenizerRule {
	static String[] stopWords = { "a", "able", "about", "across", "after", "all",
			"almost", "also", "am", "among", "an", "and", "any", "are",
			"as", "at", "be", "because", "been", "but", "by", "can",
			"cannot", "could", "dear", "did", "do", "does", "either",
			"else", "ever", "every", "for", "from", "get", "got", "had",
			"has", "have", "he", "her", "hers", "him", "his", "how",
			"however", "i", "if", "in", "into", "is", "it", "its", "just",
			"least", "let", "like", "likely", "may", "me", "might", "most",
			"must", "my", "neither", "no", "nor", "not", "of", "off",
			"often", "on", "only", "or", "other", "our", "own", "rather",
			"said", "say", "says", "she", "should", "since", "so", "some",
			"than", "that", "the", "their", "them", "then", "there",
			"these", "they", "this", "tis", "to", "too", "twas", "us",
			"wants", "was", "we", "were", "what", "when", "where", "which",
			"while", "who", "whom", "why", "will", "with", "would", "yet",
			"you", "your" };	
	static HashSet<String> stopWordSet = new HashSet<String>(Arrays.asList(stopWords));
	// Stop Words:
	// a,able,about,across,after,all,almost,also,am,among,an,and,any,are,as,at,be,because,been,but,by,can,cannot,could,dear,did,do,does,either,else,ever,every,for,from,get,got,had,has,have,he,her,hers,him,his,how,however,i,if,in,into,is,it,its,just,least,let,like,likely,may,me,might,most,must,my,neither,no,nor,not,of,off,often,on,only,or,other,our,own,rather,said,say,says,she,should,since,so,some,than,that,the,their,them,then,there,these,they,this,tis,to,too,twas,us,wants,was,we,were,what,when,where,which,while,who,whom,why,will,with,would,yet,you,your
	
	public void apply(TokenStream stream) throws TokenizerException {
		stream.reset();
		while (stream.hasNext()) {
			String token = stream.next();
			if (stopWordSet.contains(token)) {
				stream.remove();
			}
		}
	}

}

package edu.buffalo.cse.ir.wikiindexer.tokenizer.rules;

import java.util.Arrays;
import java.util.HashSet;

import edu.buffalo.cse.ir.wikiindexer.tokenizer.TokenStream;
import edu.buffalo.cse.ir.wikiindexer.tokenizer.TokenizerException;
import edu.buffalo.cse.ir.wikiindexer.tokenizer.rules.TokenizerRule.RULENAMES;

@RuleClass(className = RULENAMES.STOPWORDS)
public class StopwordsDefault implements TokenizerRule {
	final static String[] stopWords = { "a", "able", "about", "across", "after", "all",
			"almost", "also", "am", "among", "an", "and", "any", "are",
			"as", "at", "be", "because", "been", "but", "by", "can",
			"cannot", "could", "dear", "did", "do", "does", "either",
			"else", "ever", "every", "for", "from", "get", "got", "had",
			"has", "have", "he", "her", "hers", "him", "his", "how",
			"however", "ƒi", "if", "in", "into", "is", "it", "its", "just",
			"least", "let", "like", "likely", "may", "me", "might", "most",
			"must", "my", "neither", "no", "nor", "not", "of", "off",
			"often", "on", "only", "or", "other", "our", "own", "rather",
			"said", "say", "says", "she", "should", "since", "so", "some",
			"than", "that", "the", "their", "them", "then", "there",
			"these", "they", "this", "tis", "to", "too", "twas", "us",
			"wants", "was", "we", "were", "what", "when", "where", "which",
			"while", "who", "whom", "why", "will", "with", "would", "yet",
			"you", "your" };	
	final static HashSet<String> stopWordSet = new HashSet<String>(Arrays.asList(stopWords));
	// Stop Words:
	// a,able,about,across,after,all,almost,also,am,among,an,and,any,are,as,at,be,because,been,but,by,can,cannot,could,dear,did,do,does,either,else,ever,every,for,from,get,got,had,has,have,he,her,hers,him,his,how,however,i,if,in,into,is,it,its,just,least,let,like,likely,may,me,might,most,must,my,neither,no,nor,not,of,off,often,on,only,or,other,our,own,rather,said,say,says,she,should,since,so,some,than,that,the,their,them,then,there,these,they,this,tis,to,too,twas,us,wants,was,we,were,what,when,where,which,while,who,whom,why,will,with,would,yet,you,your
	
	public void apply(TokenStream stream) throws TokenizerException {
		stream.reset();
		while (stream.hasNext()) {
			String token = stream.next();
			if (stopWordSet.contains(token)) {
				stream.remove();
			}
		}
	}

}

add final & static for effective