<commit_before>package fr.liglab.lcm.internals.nomaps;

import java.util.Iterator;
import java.util.concurrent.atomic.AtomicInteger;

import fr.liglab.lcm.internals.FrequentsIterator;
import fr.liglab.lcm.internals.TransactionReader;
import fr.liglab.lcm.util.ItemsetsFactory;

/**
 * Item counting and all other stats available from a single pass
 * 
 * Once constructed, all of these are available in public fields
 */
public class DatasetCountersRenamer {

	/**
	 * Anything with a support count below this value will be considered
	 * unfrequent (and, sooner or later, removed)
	 */
	public final int minSup;

	public AllFrequentsIterator extensionsIterator;
	
	/**
	 * How many transactions have been counted
	 */
	public final int transactionsCount;
	
	/**
	 * Values of supportCounts, summed
	 */
	public final int supportsSum;
	public final int distinctTransactionLengthSum;
	/**
	 * Support count for each item having a support count in [minSup; 100% [
	 */
	public int[] supportCounts;
	public int[] distinctTransactionsCounts;

	/**
	 * Items found to have a 100% support count
	 */
	public final int[] closure;

	/**
	 * items having a support count in [minSup; 100% [, sorted
	 */
	public final int nbFrequents;
	public int[] reverseRenaming;
	public int[] renaming;

	private int maxFrequent;

	private final int extension;

	DatasetCountersRenamer(int minimumSupport, Iterator<TransactionReader> transactions, int maxItem) {
		this(minimumSupport, transactions, Integer.MAX_VALUE, null, maxItem, false);
	}

	/**
	 * Counts items appearing in provided transactions
	 * 
	 * @param minimumSupport
	 * @param transactions
	 * @param ingoreItem
	 *            avoids building an array when there's only extension to be
	 *            ignored
	 * @param ignoreItems
	 *            (may be null) items that may appear in transactions but should
	 *            to be counted
	 */
	DatasetCountersRenamer(int minimumSupport, Iterator<TransactionReader> transactions, int extension,
			int[] ignoredItems, final int maxItem, boolean compactRebase) {
		// fix fake extensions
		this.extension = Math.min(extension, maxItem + 1);
		this.minSup = minimumSupport;
		int lineCount = 0;
		this.supportCounts = new int[maxItem];
		this.distinctTransactionsCounts = new int[maxItem];
		while (transactions.hasNext()) {
			TransactionReader transaction = transactions.next();
			int weight = transaction.getTransactionSupport();
			lineCount++;

			while (transaction.hasNext()) {
				int item = transaction.next();
				this.supportCounts[item] += weight;
				this.distinctTransactionsCounts[item]++;
			}
		}

		this.transactionsCount = lineCount;
		if (extension != Integer.MAX_VALUE) {
			this.supportCounts[extension] = 0;<commit_msg>now DatasetCountersRenamer.transactionsCount is the sum of counted transactions' weights<commit_after>package fr.liglab.lcm.internals.nomaps;

import java.util.Iterator;
import java.util.concurrent.atomic.AtomicInteger;

import fr.liglab.lcm.internals.FrequentsIterator;
import fr.liglab.lcm.internals.TransactionReader;
import fr.liglab.lcm.util.ItemsetsFactory;

/**
 * Item counting and all other stats available from a single pass
 * 
 * Once constructed, all of these are available in public fields
 */
public class DatasetCountersRenamer {

	/**
	 * Anything with a support count below this value will be considered
	 * unfrequent (and, sooner or later, removed)
	 */
	public final int minSup;

	public AllFrequentsIterator extensionsIterator;
	
	/**
	 * Sum of counted transactions' weights
	 */
	public final int transactionsCount;
	
	/**
	 * Values of supportCounts, summed
	 */
	public final int supportsSum;
	public final int distinctTransactionLengthSum;
	/**
	 * Support count for each item having a support count in [minSup; 100% [
	 */
	public int[] supportCounts;
	public int[] distinctTransactionsCounts;

	/**
	 * Items found to have a 100% support count
	 */
	public final int[] closure;

	/**
	 * items having a support count in [minSup; 100% [, sorted
	 */
	public final int nbFrequents;
	public int[] reverseRenaming;
	public int[] renaming;

	private int maxFrequent;

	private final int extension;

	DatasetCountersRenamer(int minimumSupport, Iterator<TransactionReader> transactions, int maxItem) {
		this(minimumSupport, transactions, Integer.MAX_VALUE, null, maxItem, false);
	}

	/**
	 * Counts items appearing in provided transactions
	 * 
	 * @param minimumSupport
	 * @param transactions
	 * @param ingoreItem
	 *            avoids building an array when there's only extension to be
	 *            ignored
	 * @param ignoreItems
	 *            (may be null) items that may appear in transactions but should
	 *            to be counted
	 */
	DatasetCountersRenamer(int minimumSupport, Iterator<TransactionReader> transactions, int extension,
			int[] ignoredItems, final int maxItem, boolean compactRebase) {
		// fix fake extensions
		this.extension = Math.min(extension, maxItem + 1);
		this.minSup = minimumSupport;
		int weightsSum = 0;
		this.supportCounts = new int[maxItem];
		this.distinctTransactionsCounts = new int[maxItem];
		while (transactions.hasNext()) {
			TransactionReader transaction = transactions.next();
			int weight = transaction.getTransactionSupport();
			
			if (transaction.hasNext()) {
				weightsSum += weight;
			}

			while (transaction.hasNext()) {
				int item = transaction.next();
				this.supportCounts[item] += weight;
				this.distinctTransactionsCounts[item]++;
			}
		}

		this.transactionsCount = weightsSum;
		if (extension != Integer.MAX_VALUE) {
			this.supportCounts[extension] = 0;
{'fraction_non_alphanumeric': 0.09112024298731464, 'fraction_numerical': 0.004288011434697159, 'mean_word_length': 3.940864960282436, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5565092', 'n_tokens_mistral': 1730, 'n_tokens_neox': 1529, 'n_words': 497}