<commit_before>#[test]
fn hextranslation() {
	let vals = vec![0, 255, 128];
	let ans = md5::to_hex_string(&vals);
	println!("{0}", ans);
	assert!(ans == "00FF80");
}

#[test]
fn empty_test() {
	use md5::*;

	let trueans = "D41D8CD98F00B204E9800998ECF8427E";
	let vals = vec![];
	let try = md5::compute(&vals);
	println!("{0}", to_hex_string(&try));
	assert!(to_hex_string(&try) == trueans);
}


extern crate core;

mod md5 {

	pub fn to_hex_string(bytes: &[u8]) -> String {
	  let strs: Vec<String> = bytes.iter().map(|b| format!("{:02X}", b)).collect();
	  strs.join("")
	}

	pub fn compute(data : &Vec<u8>) -> [u8;16] {

		use core::ops::Not;		

 		let k : [u32; 64] = [
	 		0xd76aa478, 0xe8c7b756, 0x242070db, 0xc1bdceee,
			0xf57c0faf, 0x4787c62a, 0xa8304613, 0xfd469501,
			0x698098d8, 0x8b44f7af, 0xffff5bb1, 0x895cd7be,
			0x6b901122, 0xfd987193, 0xa679438e, 0x49b40821,
			0xf61e2562, 0xc040b340, 0x265e5a51, 0xe9b6c7aa,
			0xd62f105d, 0x02441453, 0xd8a1e681, 0xe7d3fbc8,
			0x21e1cde6, 0xc33707d6, 0xf4d50d87, 0x455a14ed,
			0xa9e3e905, 0xfcefa3f8, 0x676f02d9, 0x8d2a4c8a,
			0xfffa3942, 0x8771f681, 0x6d9d6122, 0xfde5380c,
			0xa4beea44, 0x4bdecfa9, 0xf6bb4b60, 0xbebfbc70,
			0x289b7ec6, 0xeaa127fa, 0xd4ef3085, 0x04881d05,
			0xd9d4d039, 0xe6db99e5, 0x1fa27cf8, 0xc4ac5665,
			0xf4292244, 0x432aff97, 0xab9423a7, 0xfc93a039,
			0x655b59c3, 0x8f0ccc92, 0xffeff47d, 0x85845dd1,
			0x6fa87e4f, 0xfe2ce6e0, 0xa3014314, 0x4e0811a1,
			0xf7537e82, 0xbd3af235, 0x2ad7d2bb, 0xeb86d391 
		];

		let s : [u32; 64] = 
		[
			7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 
			17, 22, 7, 12, 17, 22, 5, 9, 14, 20, 5, 
			9, 14, 20, 5, 9, 14, 20, 5, 9, 14, 20, 4, 
			11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23, 
			4, 11, 16, 23, 6, 10, 15, 21, 6, 10, 15, 21, 
			6, 10, 15, 21, 6, 10, 15, 21 
		];
		
		let mut a0 : u32 = 0x67452301;   //a
		let mut b0 : u32 = 0xefcdab89;   //b
		let mut c0 : u32 = 0x98badcfe;   //c
		let mut d0 : u32 = 0x10325476;   //d


		let mut our_data = data.clone();

		our_data.push(0b10000000);
		let mod512 = our_data.len() % 64;
		let padding_length = 
			if mod512 > 56 
				{ 64-mod512+56 } 
			else 	
				{ 56 - mod512 };

		for _ in 0..padding_length {
			our_data.push(0u8);
		}

		let size : u64 = 8*data.len() as u64;
		for i in (0..8).rev() {
			our_data.push((size.wrapping_shr(8*i) % 256) as u8);
		}

		assert!(our_data.len() % 64 == 0);<commit_msg>Fixed endian of the length encoding<commit_after>#[test]
fn hextranslation() {
	let vals = vec![0, 255, 128];
	let ans = md5::to_hex_string(&vals);
	println!("{0}", ans);
	assert!(ans == "00FF80");
}

#[test]
fn empty_test() {
	use md5::*;

	let trueans = "D41D8CD98F00B204E9800998ECF8427E";
	let vals = vec![];
	let try = md5::compute(&vals);
	println!("{0}", to_hex_string(&try));
	assert!(to_hex_string(&try) == trueans);
}

#[test]
fn SevenHundredAs()
{
	use md5::*;

	let expected = "C04C6D6896853D32B720D69A6027E6BE";
	let A = 'A' as u8;
	let vals = vec![A; 700];
	let try = md5::compute(&vals);
	println!("{0}", to_hex_string(&try));
	assert!(to_hex_string(&try) == expected);
}


extern crate core;

mod md5 {

	pub fn to_hex_string(bytes: &[u8]) -> String {
	  let strs: Vec<String> = bytes.iter().map(|b| format!("{:02X}", b)).collect();
	  strs.join("")
	}

	pub fn compute(data : &Vec<u8>) -> [u8;16] {

		use core::ops::Not;		

 		let k : [u32; 64] = [
	 		0xd76aa478, 0xe8c7b756, 0x242070db, 0xc1bdceee,
			0xf57c0faf, 0x4787c62a, 0xa8304613, 0xfd469501,
			0x698098d8, 0x8b44f7af, 0xffff5bb1, 0x895cd7be,
			0x6b901122, 0xfd987193, 0xa679438e, 0x49b40821,
			0xf61e2562, 0xc040b340, 0x265e5a51, 0xe9b6c7aa,
			0xd62f105d, 0x02441453, 0xd8a1e681, 0xe7d3fbc8,
			0x21e1cde6, 0xc33707d6, 0xf4d50d87, 0x455a14ed,
			0xa9e3e905, 0xfcefa3f8, 0x676f02d9, 0x8d2a4c8a,
			0xfffa3942, 0x8771f681, 0x6d9d6122, 0xfde5380c,
			0xa4beea44, 0x4bdecfa9, 0xf6bb4b60, 0xbebfbc70,
			0x289b7ec6, 0xeaa127fa, 0xd4ef3085, 0x04881d05,
			0xd9d4d039, 0xe6db99e5, 0x1fa27cf8, 0xc4ac5665,
			0xf4292244, 0x432aff97, 0xab9423a7, 0xfc93a039,
			0x655b59c3, 0x8f0ccc92, 0xffeff47d, 0x85845dd1,
			0x6fa87e4f, 0xfe2ce6e0, 0xa3014314, 0x4e0811a1,
			0xf7537e82, 0xbd3af235, 0x2ad7d2bb, 0xeb86d391 
		];

		let s : [u32; 64] = 
		[
			7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 
			17, 22, 7, 12, 17, 22, 5, 9, 14, 20, 5, 
			9, 14, 20, 5, 9, 14, 20, 5, 9, 14, 20, 4, 
			11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23, 
			4, 11, 16, 23, 6, 10, 15, 21, 6, 10, 15, 21, 
			6, 10, 15, 21, 6, 10, 15, 21 
		];
		
		let mut a0 : u32 = 0x67452301;   //a
		let mut b0 : u32 = 0xefcdab89;   //b
		let mut c0 : u32 = 0x98badcfe;   //c
		let mut d0 : u32 = 0x10325476;   //d


		let mut our_data = data.clone();

		our_data.push(0b10000000);
		let mod512 = our_data.len() % 64;
		let padding_length = 
			if mod512 > 56 
				{ 64-mod512+56 } 
			else 	
				{ 56 - mod512 };

		for _ in 0..padding_length {
			our_data.push(0u8);
		}

		let size : u64 = 8*data.len() as u64;
		for i in (0..8) {
			our_data.push((size.wrapping_shr(8*i) % 256) as u8);
		}

		assert!(our_data.len() % 64 == 0);
{'fraction_non_alphanumeric': 0.1608738828202582, 'fraction_numerical': 0.2516385302879841, 'mean_word_length': 3.5740236148955495, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '556017', 'n_tokens_mistral': 3518, 'n_tokens_neox': 2588, 'n_words': 540}