<commit_before>package cc.mallet.util;

import cc.mallet.types.*;
import gnu.trove.*;

import java.util.Formatter;
import java.util.Locale;
import java.util.logging.*;
import java.io.*;

import java.text.NumberFormat;

public class FeatureCountTool {

	protected static Logger logger = MalletLogger.getLogger(FeatureCountTool.class.getName());

	static cc.mallet.util.CommandOption.String inputFile = new cc.mallet.util.CommandOption.String
		(FeatureCountTool.class, "input", "FILENAME", true, null,
		 "Filename for the input instance list", null);
	
	double[] featureCounts;
	InstanceList instances;
	int numFeatures;
	int[] documentFrequencies;

	public FeatureCountTool (InstanceList instances) {
		this.instances = instances;
		numFeatures = instances.getDataAlphabet().size();

		featureCounts = new double[numFeatures];
		documentFrequencies = new int[numFeatures];
	}

	public void count() {

		TIntIntHashMap docCounts = new TIntIntHashMap();
		
		int index = 0;

		if (instances.size() == 0) { 
			logger.info("Instance list is empty");
			return;
		}

		if (instances.get(0).getData() instanceof FeatureSequence) {

			for (Instance instance: instances) {
				FeatureSequence features = (FeatureSequence) instance.getData();
				
				for (int i=0; i<features.getLength(); i++) {
					docCounts.adjustOrPutValue(features.getIndexAtPosition(i), 1, 1);
				}
				
				int[] keys = docCounts.keys();
				for (int i = 0; i < keys.length - 1; i++) {
					int feature = keys[i];
					featureCounts[feature] += docCounts.get(feature);
					documentFrequencies[feature]++;
				}
				
				docCounts = new TIntIntHashMap();
				
				index++;
				if (index % 1000 == 0) { System.err.println(index); }
			}
		}
		else if (instances.get(0).getData() instanceof FeatureVector) {
			
			for (Instance instance: instances) {
				FeatureVector features = (FeatureVector) instance.getData();
				
				for (int location = 0; location < features.numLocations(); location++) {
					int feature = features.indexAtLocation(location);
					double value = features.valueAtLocation(location);

					documentFrequencies[feature]++;
					featureCounts[feature] += value;
				}

				index++;
				if (index % 1000 == 0) { System.err.println(index); }
			}
		}
		else {
			logger.info("Unsupported data class: " + instances.get(0).getData().getClass().getName());
		}
	}

	public void printCounts() {
		
		Alphabet alphabet = instances.getDataAlphabet();

		NumberFormat nf = NumberFormat.getInstance();
		nf.setMinimumFractionDigits(0);
		nf.setMaximumFractionDigits(6);
		nf.setGroupingUsed(false);

		for (int feature = 0; feature < numFeatures; feature++) {

			Formatter formatter = new Formatter(new StringBuilder(), Locale.US);
			
			formatter.format("%s\t%s\t%d", 
							 alphabet.lookupObject(feature).toString(),
							 nf.format(featureCounts[feature]), documentFrequencies[feature]);

			System.out.println(formatter);
		}
		
	}

	public static void main (String[] args) throws Exception {
		CommandOption.setSummary (FeatureCountTool.class,
								  "Print feature counts and instances per feature (eg document frequencies) in an instance list");
		CommandOption.process (FeatureCountTool.class, args);

		InstanceList instances = InstanceList.load (new File(inputFile.value));
		FeatureCountTool counter = new FeatureCountTool(instances);
		counter.count();
		counter.printCounts();
	}


}<commit_msg>new methods, not skipping last token<commit_after>package cc.mallet.util;

import cc.mallet.types.*;
import gnu.trove.*;

import java.util.Formatter;
import java.util.Locale;
import java.util.logging.*;
import java.io.*;

import java.text.NumberFormat;

public class FeatureCountTool {

	protected static Logger logger = MalletLogger.getLogger(FeatureCountTool.class.getName());

	static cc.mallet.util.CommandOption.String inputFile = new cc.mallet.util.CommandOption.String
		(FeatureCountTool.class, "input", "FILENAME", true, null,
		 "Filename for the input instance list", null);
	
	double[] featureCounts;
	InstanceList instances;
	int numFeatures;
	int[] documentFrequencies;

	public FeatureCountTool (InstanceList instances) {
		this.instances = instances;
		numFeatures = instances.getDataAlphabet().size();

		featureCounts = new double[numFeatures];
		documentFrequencies = new int[numFeatures];
	}

	public double[] getFeatureCounts() {
		return featureCounts;
	}

	public int[] getDocumentFrequencies() {
		return documentFrequencies;
	}

	public void count() {

		TIntIntHashMap docCounts = new TIntIntHashMap();
		
		int index = 0;

		if (instances.size() == 0) { 
			logger.info("Instance list is empty");
			return;
		}

		if (instances.get(0).getData() instanceof FeatureSequence) {

			for (Instance instance: instances) {
				FeatureSequence features = (FeatureSequence) instance.getData();
				
				for (int i=0; i<features.getLength(); i++) {
					docCounts.adjustOrPutValue(features.getIndexAtPosition(i), 1, 1);
				}
				
				int[] keys = docCounts.keys();
				for (int i = 0; i < keys.length; i++) {
					int feature = keys[i];
					featureCounts[feature] += docCounts.get(feature);
					documentFrequencies[feature]++;
				}
				
				docCounts = new TIntIntHashMap();
				
				index++;
				if (index % 1000 == 0) { System.err.println(index); }
			}
		}
		else if (instances.get(0).getData() instanceof FeatureVector) {
			
			for (Instance instance: instances) {
				FeatureVector features = (FeatureVector) instance.getData();
				
				for (int location = 0; location < features.numLocations(); location++) {
					int feature = features.indexAtLocation(location);
					double value = features.valueAtLocation(location);

					documentFrequencies[feature]++;
					featureCounts[feature] += value;
				}

				index++;
				if (index % 1000 == 0) { System.err.println(index); }
			}
		}
		else {
			logger.info("Unsupported data class: " + instances.get(0).getData().getClass().getName());
		}
	}

	public void printCounts() {
		
		Alphabet alphabet = instances.getDataAlphabet();

		NumberFormat nf = NumberFormat.getInstance();
		nf.setMinimumFractionDigits(0);
		nf.setMaximumFractionDigits(6);
		nf.setGroupingUsed(false);

		for (int feature = 0; feature < numFeatures; feature++) {

			Formatter formatter = new Formatter(new StringBuilder(), Locale.US);
			
			formatter.format("%s\t%s\t%d", 
							 alphabet.lookupObject(feature).toString(),
							 nf.format(featureCounts[feature]), documentFrequencies[feature]);

			System.out.println(formatter);
		}
		
	}

	public static void main (String[] args) throws Exception {
		CommandOption.setSummary (FeatureCountTool.class,
								  "Print feature counts and instances per feature (eg document frequencies) in an instance list");
		CommandOption.process (FeatureCountTool.class, args);

		InstanceList instances = InstanceList.load (new File(inputFile.value));
		FeatureCountTool counter = new FeatureCountTool(instances);
		counter.count();
		counter.printCounts();
	}


}
{'fraction_non_alphanumeric': 0.12704800229951135, 'fraction_numerical': 0.006754814601897097, 'mean_word_length': 4.010079193664507, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2353385', 'n_tokens_mistral': 2485, 'n_tokens_neox': 2051, 'n_words': 484}