<commit_before>package fr.liglab.lcm.internals;

import fr.liglab.lcm.util.ItemsetsFactory;
import gnu.trove.iterator.TIntIntIterator;
import gnu.trove.map.TIntIntMap;
import gnu.trove.map.hash.TIntIntHashMap;
import gnu.trove.set.TIntSet;

import java.util.Arrays;
import java.util.Iterator;

/**
 * Item counting and all other stats available from a single pass
 * 
 * Once constructed, all of these are available in public fields
 */
public class DatasetCounters {
	
	/**
	 * Anything with a support count below this value will be considered 
	 * unfrequent (and, sooner or later, removed)
	 */
	public final int minSup;
	
	/**
	 * How many transactions have been counted
	 */
	public final int transactionsCount;
	
	/**
	 * How many items have been read from inputted transactions
	 */
	public final int itemsRead;
	
	/**
	 * Values of supportCounts, summed
	 */
	public final int supportsSum;
	
	/**
	 * Support count for each item having a support count in [minSup; 100% [
	 */
	public final TIntIntMap supportCounts;
	
	/**
	 * Items found to have a 100% support count
	 */
	public final int[] closure;
	
	
	DatasetCounters(int minimumSupport, Iterator<TransactionReader> transactions) {
		this(minimumSupport, transactions, null);
	}
	
	/**
	 * Counts items appearing in provided transactions
	 * @param minimumSupport
	 * @param transactions
	 * @param ignoreItems (may be null) items that may appear in transactions but should to be counted 
	 */
	DatasetCounters(int minimumSupport, Iterator<TransactionReader> transactions, int[] ignoredItems) {
		this.minSup = minimumSupport;
		
		int lineCount = 0;
		int itemCount = 0;
		this.supportCounts = new TIntIntHashMap();
		
		while (transactions.hasNext()) {
			TransactionReader transaction = transactions.next();
			int weight = transaction.getTransactionSupport();
			lineCount++;
			
			while (transaction.hasNext()) {
				itemCount++;
				int item = transaction.next();
				supportCounts.adjustOrPutValue(item, weight, weight);
			}
		}
		
		this.transactionsCount = lineCount;
		this.itemsRead = itemCount;
		
		if (ignoredItems != null) {
			for (int item : ignoredItems) {
				supportCounts.remove(item);
			}
		}
		
		ItemsetsFactory builder = new ItemsetsFactory();
		int remainingSupportsSum = 0;
		
		for (TIntIntIterator count = supportCounts.iterator(); count.hasNext();){
			count.advance();
			
			if (count.value() < minimumSupport) {
				count.remove();
			} else if (count.value() == this.transactionsCount) {
				builder.add(count.key());
				count.remove();
			} else {
				remainingSupportsSum += count.value();
			}
		}
		
		this.closure = builder.get();
		this.supportsSum = remainingSupportsSum;
	}
	
	/**
	 * @return items having a support count in [minSup; 100% [ (sorted in ascending)
	 */
	public final int[] getSortedFrequents() {
		int[] frequents = this.supportCounts.keys();
		Arrays.sort(frequents);
		return frequents;
	}
	
	/**
	 * @return item having a support count in [minSup; 100% [
	 */
	public final TIntSet getFrequents() {
		return this.supportCounts.keySet();
	}
}<commit_msg>frequent items array is now a public final field of DatasetCounters<commit_after>package fr.liglab.lcm.internals;

import fr.liglab.lcm.util.ItemsetsFactory;
import gnu.trove.iterator.TIntIntIterator;
import gnu.trove.iterator.TIntIterator;
import gnu.trove.map.TIntIntMap;
import gnu.trove.map.hash.TIntIntHashMap;
import gnu.trove.set.TIntSet;

import java.util.Arrays;
import java.util.Iterator;
import java.util.concurrent.atomic.AtomicInteger;

/**
 * Item counting and all other stats available from a single pass
 * 
 * Once constructed, all of these are available in public fields
 */
public class DatasetCounters {
	
	/**
	 * Anything with a support count below this value will be considered 
	 * unfrequent (and, sooner or later, removed)
	 */
	public final int minSup;
	
	/**
	 * How many transactions have been counted
	 */
	public final int transactionsCount;
	
	/**
	 * How many items have been read from inputted transactions
	 */
	public final int itemsRead;
	
	/**
	 * Values of supportCounts, summed
	 */
	public final int supportsSum;
	
	/**
	 * Support count for each item having a support count in [minSup; 100% [
	 */
	public final TIntIntMap supportCounts;
	
	/**
	 * Items found to have a 100% support count
	 */
	public final int[] closure;
	
	/**
	 * items having a support count in [minSup; 100% [, sorted
	 */
	public final int[] sortedFrequents;
	
	
	DatasetCounters(int minimumSupport, Iterator<TransactionReader> transactions) {
		this(minimumSupport, transactions, null);
	}
	
	/**
	 * Counts items appearing in provided transactions
	 * @param minimumSupport
	 * @param transactions
	 * @param ignoreItems (may be null) items that may appear in transactions but should to be counted 
	 */
	DatasetCounters(int minimumSupport, Iterator<TransactionReader> transactions, int[] ignoredItems) {
		this.minSup = minimumSupport;
		
		int lineCount = 0;
		int itemCount = 0;
		this.supportCounts = new TIntIntHashMap();
		
		while (transactions.hasNext()) {
			TransactionReader transaction = transactions.next();
			int weight = transaction.getTransactionSupport();
			lineCount++;
			
			while (transaction.hasNext()) {
				itemCount++;
				int item = transaction.next();
				supportCounts.adjustOrPutValue(item, weight, weight);
			}
		}
		
		this.transactionsCount = lineCount;
		this.itemsRead = itemCount;
		
		if (ignoredItems != null) {
			for (int item : ignoredItems) {
				supportCounts.remove(item);
			}
		}
		
		ItemsetsFactory builder = new ItemsetsFactory();
		int remainingSupportsSum = 0;
		
		for (TIntIntIterator count = supportCounts.iterator(); count.hasNext();){
			count.advance();
			
			if (count.value() < minimumSupport) {
				count.remove();
			} else if (count.value() == this.transactionsCount) {
				builder.add(count.key());
				count.remove();
			} else {
				remainingSupportsSum += count.value();
			}
		}
		
		this.closure = builder.get();
		this.supportsSum = remainingSupportsSum;
		
		this.sortedFrequents = this.supportCounts.keys();
		Arrays.sort(this.sortedFrequents);
	}
	
	/**
	 * @return item having a support count in [minSup; 100% [
	 */
	public final TIntSet getFrequents() {
		return this.supportCounts.keySet();
	}
	
	
	/**
	 * Thread-safe iterator over sortedFrequents
	 */
	public class FrequentsIterator implements TIntIterator {
		private final AtomicInteger index;
		private final int max;
		
		/**
		 * will provide an iterator on frequent items in [0,to[
		 */
		public FrequentsIterator(int to) {
			this.index = new AtomicInteger(0);
			
			int toIndex = Arrays.binarySearch(sortedFrequents, to);
			if (toIndex < 0) {
				// binarySearch returns -(insertion_point)-1
				// where insertion_point == index of first element greater OR
				// a.length
				this.max = -toIndex - 1;
			} else {
				this.max = toIndex;
			}
		}
		
		@Override
		public final boolean hasNext() {
			return this.index.get() < this.max;
		}
		
		@Override
		public final int next() {
			return sortedFrequents[this.index.getAndIncrement()];
		}
		
		@Override
		public final void remove() {}
	}
}
{'fraction_non_alphanumeric': 0.10746812386156648, 'fraction_numerical': 0.00490402129746392, 'mean_word_length': 3.540712468193384, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6125068', 'n_tokens_mistral': 2492, 'n_tokens_neox': 2091, 'n_words': 619}