<commit_before>package it.unimi.di.law.bubing.tool;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.net.Inet4Address;
import java.nio.charset.StandardCharsets;
import java.util.function.LongConsumer;

import it.unimi.di.law.bubing.frontier.VisitState;
import it.unimi.di.law.bubing.util.Util;
import it.unimi.dsi.fastutil.io.FastBufferedInputStream;
import it.unimi.dsi.fastutil.longs.Long2LongOpenHashMap;
import it.unimi.dsi.fastutil.objects.Object2LongOpenHashMap;

public class AnalyzeWorkbench {

	@SuppressWarnings("boxing")
	public static void main(String[] args) throws ClassNotFoundException, IOException {
		final ObjectInputStream workbenchStream = new ObjectInputStream(new FastBufferedInputStream(new FileInputStream(new File(args[0]))));

		final long workbenchSize = workbenchStream.readLong();
		long w = workbenchSize;
		final Object2LongOpenHashMap<byte[]> count = new Object2LongOpenHashMap<>();
		count.defaultReturnValue(0);
		while(w-- != 0) {
			final VisitState visitState = (VisitState)workbenchStream.readObject();
			final boolean nonNullWorkbenchEntry = workbenchStream.readBoolean();
			if (nonNullWorkbenchEntry) {
				final byte[] ipAddress = Util.readByteArray(workbenchStream);
				count.addTo(ipAddress, 1);
				System.err.print(new String(visitState.schemeAuthority, StandardCharsets.US_ASCII));
				System.err.print('\t');
				System.err.println(Inet4Address.getByAddress(ipAddress));
			}
		}

		final Long2LongOpenHashMap count2Freq = new Long2LongOpenHashMap();
		count2Freq.defaultReturnValue(0);
		count.values().forEach((LongConsumer)(x -> { count2Freq.addTo(x, 1); }));
		count2Freq.forEach((x,y)-> { System.out.println(x + '\t' + y); });
	}

}<commit_msg>Tweaking<commit_after>package it.unimi.di.law.bubing.tool;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.net.Inet4Address;
import java.nio.charset.StandardCharsets;
import java.util.function.LongConsumer;

import it.unimi.di.law.bubing.frontier.VisitState;
import it.unimi.di.law.bubing.util.Util;
import it.unimi.dsi.fastutil.io.FastBufferedInputStream;
import it.unimi.dsi.fastutil.ints.Int2LongOpenHashMap;
import it.unimi.dsi.fastutil.longs.Long2LongOpenHashMap;
import com.google.common.primitives.Ints;

public class AnalyzeWorkbench {

	@SuppressWarnings("boxing")
	public static void main(String[] args) throws ClassNotFoundException, IOException {
		final ObjectInputStream workbenchStream = new ObjectInputStream(new FastBufferedInputStream(new FileInputStream(new File(args[0]))));

		final long workbenchSize = workbenchStream.readLong();
		long w = workbenchSize;
		Int2LongOpenHashMap ip2count = new Int2LongOpenHashMap();
		ip2count.defaultReturnValue(0);
		while(w-- != 0) {
			final VisitState visitState = (VisitState)workbenchStream.readObject();
			final boolean nonNullWorkbenchEntry = workbenchStream.readBoolean();
			if (nonNullWorkbenchEntry) {
				byte[] ipAddress = Util.readByteArray(workbenchStream);
				ip2count.addTo(Ints.fromByteArray(ipAddress), 1);
				System.err.print(new String(visitState.schemeAuthority, StandardCharsets.US_ASCII));
				System.err.print('\t');
				System.err.println(Inet4Address.getByAddress(ipAddress));
			}
		}
		
		Long2LongOpenHashMap count2Freq = new Long2LongOpenHashMap();
		count2Freq.defaultReturnValue(0);
		ip2count.values().forEach((LongConsumer)(x -> { count2Freq.addTo(x, 1); }));
		count2Freq.forEach((x,y)-> { System.out.println(x + "\t" + y); });
	}
}
{'fraction_non_alphanumeric': 0.12430323299888517, 'fraction_numerical': 0.011148272017837236, 'mean_word_length': 5.875478927203065, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4638932', 'n_tokens_mistral': 1238, 'n_tokens_neox': 1170, 'n_words': 189}