<commit_before>package axiom.objectmodel.dom;

import java.io.IOException;
import java.io.Reader;
import java.util.regex.*;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;

public class UrlAnalyzer extends Analyzer {
	public TokenStream tokenStream(String fieldName, final Reader reader) {
		return new TokenStream() {
			private final Pattern stripTokens = Pattern.compile("^(http:/|www|com|org|net)");
            private final Pattern endTokens = Pattern.compile("(\\.|/|-|_|\\?)$");
            private boolean done = false;
            public Token next() throws IOException {
            	if(!done){
	                final char[] buffer = new char[1];
	                StringBuffer sb = new StringBuffer();
	                int length = 0;
	                    
	                Matcher matcher = endTokens.matcher(sb);
	                boolean found = matcher.find();
	                
	                while (!found && (length = reader.read(buffer)) != -1) {
	                	sb.append(buffer, 0, length);
	                	Matcher startMatcher = stripTokens.matcher(sb);
	                	if(startMatcher.matches()){
	                		// strip prefix
	                		String tmp = sb.toString();
	                		sb = new StringBuffer(tmp.replaceFirst("^(http:/|www|com|org|net)", ""));
	                	}
	                	matcher = endTokens.matcher(sb);
	                	found = matcher.find();	  
	                	
	                	if(found){
		                	final String text = sb.toString().substring(0, matcher.end()-1);
		                	int len = text.length();
		                	if(len > 0){
		                		// matched a token
		                		return new Token(text, 0, len);
		                	} else {
		                		// only contains a stop token, continue reading
		                		sb = new StringBuffer();
		                		found = false;
		                	}
		                }
	                }
                	// at end of string
	                done = true;
	                final String value = sb.toString();
	                return new Token(value, 0, value.length());
            	}
            	return null;<commit_msg>fix case issue in urlanalyzer<commit_after>package axiom.objectmodel.dom;

import java.io.IOException;
import java.io.Reader;
import java.util.regex.*;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;

public class UrlAnalyzer extends Analyzer {
	public TokenStream tokenStream(String fieldName, final Reader reader) {
		return new TokenStream() {
			private final Pattern stripTokens = Pattern.compile("^(http:/|www|com|org|net)");
            private final Pattern endTokens = Pattern.compile("(\\.|/|-|_|\\?)$");
            private boolean done = false;
            public Token next() throws IOException {
            	if(!done){
	                final char[] buffer = new char[1];
	                StringBuffer sb = new StringBuffer();
	                int length = 0;
	                    
	                Matcher matcher = endTokens.matcher(sb);
	                boolean found = matcher.find();
	                
	                while (!found && (length = reader.read(buffer)) != -1) {
	                	sb.append(buffer, 0, length);
	                	Matcher startMatcher = stripTokens.matcher(sb);
	                	if(startMatcher.matches()){
	                		// strip prefix
	                		String tmp = sb.toString();
	                		sb = new StringBuffer(tmp.replaceFirst("^(http:/|www|com|org|net)", ""));
	                	}
	                	matcher = endTokens.matcher(sb);
	                	found = matcher.find();	  
	                	
	                	if(found){
		                	final String text = sb.toString().substring(0, matcher.end()-1).toLowerCase();
		                	int len = text.length();
		                	if(len > 0){
		                		// matched a token
		                		System.out.println("-- token: ["+text+"]");
		                		return new Token(text, 0, len);
		                	} else {
		                		// only contains a stop token, continue reading
		                		sb = new StringBuffer();
		                		found = false;
		                	}
		                }
	                }
                	// at end of string
	                done = true;
	                final String value = sb.toString().toLowerCase();
	                System.out.println("-- token: ["+value+"]");
	                return new Token(value, 0, value.length());
            	}
            	return null;
{'fraction_non_alphanumeric': 0.11205432937181664, 'fraction_numerical': 0.0038200339558573855, 'mean_word_length': 1.3683417085427136, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4256413', 'n_tokens_mistral': 1317, 'n_tokens_neox': 1169, 'n_words': 289}