<commit_before>package com.vaguehope.onosendai.util;

import java.io.UnsupportedEncodingException;
import java.math.BigInteger;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;

public final class HashHelper {

	private HashHelper () {
		throw new AssertionError();
	}

	public static BigInteger md5String (final String s) {
		final MessageDigest md = MD_MD5_FACTORY.get();
		md.update(getBytes(s), 0, s.length());
		return new BigInteger(1, md.digest());
	}

	private static byte[] getBytes (final String s) {
		try {
			return s.getBytes("UTF-8");
		}
		catch (UnsupportedEncodingException e) {
			throw new IllegalStateException(e.getMessage(), e);
		}
	}

	/**
	 * MessageDigest.getInstance("MD5") can take up to a second, so using this
	 * to cache it and improve performance.
	 */
	private static final ThreadLocal<MessageDigest> MD_MD5_FACTORY = new ThreadLocal<MessageDigest>() {
		@Override
		protected MessageDigest initialValue () {
			try {
				final MessageDigest md = MessageDigest.getInstance("MD5");
				md.reset();
				return md;
			}
			catch (final NoSuchAlgorithmException e) {
				throw new IllegalStateException("JVM is missing MD5.", e);
			}<commit_msg>Fix possible issue when MD5ing multibyte strings.<commit_after>package com.vaguehope.onosendai.util;

import java.io.UnsupportedEncodingException;
import java.math.BigInteger;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;

public final class HashHelper {

	private HashHelper () {
		throw new AssertionError();
	}

	public static BigInteger md5String (final String s) {
		final MessageDigest md = MD_MD5_FACTORY.get();
		md.update(getBytes(s));
		return new BigInteger(1, md.digest());
	}

	private static byte[] getBytes (final String s) {
		try {
			return s.getBytes("UTF-8");
		}
		catch (UnsupportedEncodingException e) {
			throw new IllegalStateException(e.getMessage(), e);
		}
	}

	/**
	 * MessageDigest.getInstance("MD5") can take up to a second, so using this
	 * to cache it and improve performance.
	 */
	private static final ThreadLocal<MessageDigest> MD_MD5_FACTORY = new ThreadLocal<MessageDigest>() {
		@Override
		protected MessageDigest initialValue () {
			try {
				final MessageDigest md = MessageDigest.getInstance("MD5");
				md.reset();
				return md;
			}
			catch (final NoSuchAlgorithmException e) {
				throw new IllegalStateException("JVM is missing MD5.", e);
			}
{'fraction_non_alphanumeric': 0.10560785918952108, 'fraction_numerical': 0.007367990176013099, 'mean_word_length': 4.39514348785872, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '934561', 'n_tokens_mistral': 827, 'n_tokens_neox': 744, 'n_words': 204}