<commit_before>package edu.gslis.textrepresentation;


import java.io.IOException;
import java.io.StringReader;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.util.CharArraySet;

import edu.gslis.indexes.IndexWrapper;
import edu.gslis.utils.KeyValuePair;
import edu.gslis.utils.ScorableComparator;
import edu.gslis.utils.Stopper;




/**
 * Simple container mapping term-count pairs grabbed from an input text.
 * 
 * @author <NAME>
 *
 */
public class FeatureVector  {
	private static StandardAnalyzer analyzer;	
	private Map<String, Double> features;
	private Stopper stopper;
	private double length = 0.0;


	// CONSTRUCTORS  
	public FeatureVector(String text, Stopper stopper) {
		this.stopper = stopper;
		
		if(stopper==null || stopper.asSet().size()==0) {
			analyzer = new StandardAnalyzer();
		} else {
			CharArraySet charArraySet = new CharArraySet(stopper.asSet(), true);
			analyzer = new StandardAnalyzer(charArraySet);
		}
		features = new HashMap<String, Double>();
		List<String> terms = this.analyze(text);
		Iterator<String> termsIt = terms.iterator();
		while(termsIt.hasNext()) {
			String term = termsIt.next();
			length += 1.0;
			Double val = (Double)features.get(term);
			if(val == null) {
				features.put(term, new Double(1.0));
			} else {
				double v = val.doubleValue() + 1.0;
				features.put(term, new Double(v));
			}
		}
	}

	public FeatureVector(Stopper stopper) {
		this.stopper = stopper;
		if(stopper==null || stopper.asSet().size()==0) {
			analyzer = new StandardAnalyzer();
		} else {
			CharArraySet charArraySet = new CharArraySet(stopper.asSet(), true);
			analyzer = new StandardAnalyzer(charArraySet);
		}
		features = new HashMap<String,Double>();
	}






	// MUTATORS

	/**
	 * Add all the terms in a string to this vector
	 * @param text a space-delimited string where we want to add each word.
	 */
	public void addText(String text) {
		List<String> terms = this.analyze(text);
		Iterator<String> termsIt = terms.iterator();
		while(termsIt.hasNext()) {
			String term = termsIt.next();		
			addTerm(term);
		}
	}

	/**
	 * Add a term to this vector.  if it's already here, increment its count.
	 * @param term
	 */
	public void addTerm(String term) {
		if(stopper != null && stopper.isStopWord(term))
			return;

		Double freq = ((Double)features.get(term));
		if(freq == null) {
			features.put(term, new Double(1.0));
		} else {
			double f = freq.doubleValue();
			features.put(term, new Double(f+1.0));
		}
		length += 1.0;
	}

	public void setTerm(String term, double weight) {
		if(stopper != null && stopper.isStopWord(term))
			return;
		length += weight;
		features.put(term, new Double(weight));
		
	}

	/**
	 * Add a term to this vector with this weight.  if it's already here, supplement its weight.
	 * @param term
	 * @param weight
	 */
	public void addTerm(String term, double weight) {
		Double w = ((Double)features.get(term));
		if(w == null) {
			features.put(term, new Double(weight));
		} else {
			double f = w.doubleValue();
			features.put(term, new Double(f+weight));
		}
		length += weight;
	}
	<commit_msg>This is why I hate copy+paste ;) (#13)<commit_after>package edu.gslis.textrepresentation;


import java.io.IOException;
import java.io.StringReader;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.util.CharArraySet;

import edu.gslis.indexes.IndexWrapper;
import edu.gslis.utils.KeyValuePair;
import edu.gslis.utils.ScorableComparator;
import edu.gslis.utils.Stopper;




/**
 * Simple container mapping term-count pairs grabbed from an input text.
 * 
 * @author <NAME>
 *
 */
public class FeatureVector  {
	private static StandardAnalyzer analyzer;	
	private Map<String, Double> features;
	private Stopper stopper;
	private double length = 0.0;


	// CONSTRUCTORS  
	public FeatureVector(String text, Stopper stopper) {
		this.stopper = stopper;
		
		if(stopper==null || stopper.asSet().size()==0) {
			analyzer = new StandardAnalyzer();
		} else {
			CharArraySet charArraySet = new CharArraySet(stopper.asSet(), true);
			analyzer = new StandardAnalyzer(charArraySet);
		}
		features = new HashMap<String, Double>();
		List<String> terms = this.analyze(text);
		Iterator<String> termsIt = terms.iterator();
		while(termsIt.hasNext()) {
			String term = termsIt.next();
			length += 1.0;
			Double val = (Double)features.get(term);
			if(val == null) {
				features.put(term, new Double(1.0));
			} else {
				double v = val.doubleValue() + 1.0;
				features.put(term, new Double(v));
			}
		}
	}

	public FeatureVector(Stopper stopper) {
		this.stopper = stopper;
		if(stopper==null || stopper.asSet().size()==0) {
			analyzer = new StandardAnalyzer();
		} else {
			CharArraySet charArraySet = new CharArraySet(stopper.asSet(), true);
			analyzer = new StandardAnalyzer(charArraySet);
		}
		features = new HashMap<String,Double>();
	}






	// MUTATORS

	/**
	 * Add all the terms in a string to this vector
	 * @param text a space-delimited string where we want to add each word.
	 */
	public void addText(String text) {
		List<String> terms = this.analyze(text);
		Iterator<String> termsIt = terms.iterator();
		while(termsIt.hasNext()) {
			String term = termsIt.next();		
			addTerm(term);
		}
	}

	/**
	 * Add a term to this vector.  if it's already here, increment its count.
	 * @param term
	 */
	public void addTerm(String term) {
		this.addTerm(term, 1.0);
	}

	public void setTerm(String term, double weight) {
		if(stopper != null && stopper.isStopWord(term))
			return;
		length += weight;
		features.put(term, new Double(weight));
		
	}

	/**
	 * Add a term to this vector with this weight.  if it's already here, supplement its weight.
	 * @param term
	 * @param weight
	 */
	public void addTerm(String term, double weight) {
		if(stopper != null && stopper.isStopWord(term))
			return;
		
		Double w = ((Double)features.get(term));
		if(w == null) {
			features.put(term, new Double(weight));
		} else {
			double f = w.doubleValue();
			features.put(term, new Double(f+weight));
		}
		length += weight;
	}
	
{'fraction_non_alphanumeric': 0.13188890504580486, 'fraction_numerical': 0.004362367311327614, 'mean_word_length': 3.895373665480427, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3018385', 'n_tokens_mistral': 2502, 'n_tokens_neox': 2239, 'n_words': 559}