<commit_before>package common;

import java.awt.Color;
import java.awt.Graphics2D;
import java.awt.RenderingHints;
import java.awt.image.BufferedImage;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.math.BigInteger;
import java.security.KeyManagementException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Enumeration;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Timer;
import java.util.TimerTask;
import java.util.WeakHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.imageio.ImageIO;
import javax.net.ssl.SSLContext;

import common.HttpUtil.SimpleDownloadProgress;

/**
 * All kinds of useful stuff that java should just have already.
 * @author gary
 *
 */
public abstract class Util {

	/**
	 * Describes an object that can filter items.
	 * @author gary
	 *
	 * @param <T> The type of object that can be filtered by this.
	 */
	public interface Filter<T> {
		/**
		 * Returns true if the item supplied passes through the filter.
		 * @param item
		 * @return
		 */
		boolean accept(T item);
	}
	
	/**
	 * Filters the collection supplied using a filter.
	 * @param collection The collection that will be filtered. It must support a modifyable iterator.
	 * @param filter The filter to use.
	 */
	public static <T> void filterList(Collection<T> collection, Filter<T> filter) throws UnsupportedOperationException {
		Iterator<T> itt = collection.iterator();
		while (itt.hasNext()) {
			T item = itt.next();
			if (!filter.accept(item)) itt.remove();
		}
	}
	
	/**
	 * Allows an interator to pretend to be an enumeration
	 * @author gary
	 *
	 */
	public static class EnumerationWrapper<T> implements Enumeration<T> {
		private Iterator<T> internal;
		public EnumerationWrapper(Iterable<T> input) {
			internal = input.iterator();
		}
		@Override
		public boolean hasMoreElements() {
			return internal.hasNext();
		}
		@Override
		public T nextElement() {
			return internal.next();
		}
	}
	
	/**
	 * Join the string representation of objects together with a delimiter between them.
	 * @param items The array of objects to join.
	 * @param delim The separator/delimiter.
	 * @return
	 */
	public static String join(Object[] items, String delim) {
		if (items.length == 0) return "";
		StringBuilder sb = new StringBuilder(items[0].toString());
		int onItem = 1;
		while (onItem != items.length) sb.append(delim).append(items[onItem++].toString());
		return sb.toString();
	}
	
	/**
	 * Given a simple quoted string removes the quotes from the beginning and end, and unescapes the contents according to standard conventions:
	 * Examples:<br>
	 * "w00t\"" -> w00t"
	 * "w0\\\"t" -> w0\"t
	 * "\\" -> \
	 * @param toUnquote
	 * @return
	 */
	public static String unquote(String toUnquote) {
		if (toUnquote.length()<2) return "";
		return toUnquote.substring(1, toUnquote.length()-1).replaceAll("\\\\\"", Matcher.quoteReplacement("\"")).replaceAll("\\\\\\\\", Matcher.quoteReplacement("\\"));
	}
	
	public static String bytesToHexString(byte[] inBytes) {
		return new BigInteger(inBytes).toString(0x10);
	}
	
	public static void copyFile(File a, File b) throws IOException {
		InputStream sa = new BufferedInputStream(new FileInputStream(a));
		writeStreamToFile(sa, b);
	}
	
	/**
	 * Writes the given input stream to a file, then closes the file and stream.
	 * Uses a temporary file to avoid trashing an existing file if interrupted part-way through.
	 * 
	 * Always closes both the file and the stream.
	 * 
	 * @param is
	 * @param file
	 * @throws IOException 
	 */
	public static void writeStreamToFile(InputStream is, File saveAs) throws IOException {
		writeStreamToFile(is, saveAs, null);
	}
	
	public static void writeStreamToFile(InputStream is, File saveAs, SimpleDownloadProgress progress) throws IOException {
		File partial = new File(saveAs.getAbsoluteFile()+".working");
		OutputStream outs = new BufferedOutputStream(new FileOutputStream(partial));
		byte[] buf = new byte[FS2Constants.ARBITRARY_BUFFER_SIZE];
		long soFar = 0;<commit_msg>Revert "More small utility improvements"<commit_after>package common;

import java.awt.Color;
import java.awt.Graphics2D;
import java.awt.RenderingHints;
import java.awt.image.BufferedImage;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.security.KeyManagementException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Enumeration;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Timer;
import java.util.TimerTask;
import java.util.WeakHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.imageio.ImageIO;
import javax.net.ssl.SSLContext;

import common.HttpUtil.SimpleDownloadProgress;

/**
 * All kinds of useful stuff that java should just have already.
 * @author gary
 *
 */
public abstract class Util {

	/**
	 * Describes an object that can filter items.
	 * @author gary
	 *
	 * @param <T> The type of object that can be filtered by this.
	 */
	public interface Filter<T> {
		/**
		 * Returns true if the item supplied passes through the filter.
		 * @param item
		 * @return
		 */
		boolean accept(T item);
	}
	
	/**
	 * Filters the collection supplied using a filter.
	 * @param collection The collection that will be filtered. It must support a modifyable iterator.
	 * @param filter The filter to use.
	 */
	public static <T> void filterList(Collection<T> collection, Filter<T> filter) throws UnsupportedOperationException {
		Iterator<T> itt = collection.iterator();
		while (itt.hasNext()) {
			T item = itt.next();
			if (!filter.accept(item)) itt.remove();
		}
	}
	
	/**
	 * Allows an interator to pretend to be an enumeration
	 * @author gary
	 *
	 */
	public static class EnumerationWrapper<T> implements Enumeration<T> {
		private Iterator<T> internal;
		public EnumerationWrapper(Iterable<T> input) {
			internal = input.iterator();
		}
		@Override
		public boolean hasMoreElements() {
			return internal.hasNext();
		}
		@Override
		public T nextElement() {
			return internal.next();
		}
	}
	
	/**
	 * Join the string representation of objects together with a delimiter between them.
	 * @param items The array of objects to join.
	 * @param delim The separator/delimiter.
	 * @return
	 */
	public static String join(Object[] items, String delim) {
		if (items.length==0) return "";
		StringBuffer buffer = new StringBuffer(items[0].toString());
		int onItem = 1;
		while (onItem != items.length) buffer.append(delim).append(items[onItem++].toString());
		return buffer.toString();
	}
	
	/**
	 * Given a simple quoted string removes the quotes from the beginning and end, and unescapes the contents according to standard conventions:
	 * Examples:<br>
	 * "w00t\"" -> w00t"
	 * "w0\\\"t" -> w0\"t
	 * "\\" -> \
	 * @param toUnquote
	 * @return
	 */
	public static String unquote(String toUnquote) {
		if (toUnquote.length()<2) return "";
		return toUnquote.substring(1, toUnquote.length()-1).replaceAll("\\\\\"", Matcher.quoteReplacement("\"")).replaceAll("\\\\\\\\", Matcher.quoteReplacement("\\"));
	}
	
	public static String bytesToHexString(byte[] inBytes) {
		StringBuilder sb = new StringBuilder();
		
		char[] hexChars = "0123456789abcdef".toCharArray();
		for (byte b : inBytes) {
			sb.append(hexChars[(b & 0xF0)>>4]);
			sb.append(hexChars[(b & 0x0F)]);
		}
		
		return sb.toString();
	}
	
	public static void copyFile(File a, File b) throws IOException {
		InputStream sa = new BufferedInputStream(new FileInputStream(a));
		writeStreamToFile(sa, b);
	}
	
	/**
	 * Writes the given input stream to a file, then closes the file and stream.
	 * Uses a temporary file to avoid trashing an existing file if interrupted part-way through.
	 * 
	 * Always closes both the file and the stream.
	 * 
	 * @param is
	 * @param file
	 * @throws IOException 
	 */
	public static void writeStreamToFile(InputStream is, File saveAs) throws IOException {
		writeStreamToFile(is, saveAs, null);
	}
	
	public static void writeStreamToFile(InputStream is, File saveAs, SimpleDownloadProgress progress) throws IOException {
		File partial = new File(saveAs.getAbsoluteFile()+".working");
		OutputStream outs = new BufferedOutputStream(new FileOutputStream(partial));
		byte[] buf = new byte[FS2Constants.ARBITRARY_BUFFER_SIZE];
		long soFar = 0;
{'fraction_non_alphanumeric': 0.11615427613191727, 'fraction_numerical': 0.005366126327557294, 'mean_word_length': 4.126647564469914, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3266017', 'n_tokens_mistral': 2866, 'n_tokens_neox': 2592, 'n_words': 814}