<commit_before>use std::cmp::{ max, min };

fn uppercase_all(base_word: &[u8]) -> Vec<u8> {
	let l = base_word.len();
	let mut v = Vec::with_capacity(l);
	let mut i = 0;

	while i < l {
		match base_word[i] {
			0...96|123...191 => {
				v.push(base_word[i]);
				i += 1;
			},
			97...122 => {
				v.push(base_word[i] ^ 32);
				i += 1;
			},
			192...223 => {
				v.push(base_word[i]);
				if i + 1 < l {
					v.push(base_word[i + 1] ^ 32);
				}
				i += 2;
			},
			224...255 => {
				v.push(base_word[i]);
				if i + 1 < l {
					v.push(base_word[i + 1]);
				}
				if i + 2 < l {
					v.push(base_word[i + 2] ^ 5);
				}
				i = i + 3;
			},
			_ => unreachable!(), // confirmed unreachable, all byte values covered
		}
	}

	v
}

fn uppercase_first(base_word: &[u8]) -> Vec<u8> {
	let l = base_word.len();

	if l == 0 {
		return Vec::new();
	}

	let mut v = Vec::with_capacity(l);
	let i;

	match base_word[0] {
		1...96|123...191 => {
			v.push(base_word[0]);
			i = 1;
		},
		97...122 => {
			v.push(base_word[0] ^ 32);
			i = 1;
		},
		192...223 => {
			v.push(base_word[0]);
			if 1 < l {
				v.push(base_word[1] ^ 32);
			}
			i = 2;
		},
		224...255 => {
			v.push(base_word[0]);
			if 1 < l {
				v.push(base_word[1]);
			}
			if 2 < l {
				v.push(base_word[2] ^ 5);
			}
			i = 3;
		},
		_ => unreachable!(), // confirmed unreachable, all byte values covered
	}

	[v, Vec::from(&base_word[i..])].concat()
}

/// @TODO transformation() SHOULD return a Result, for cases where a base_word is passed in
///       for example with length 3, and OmitFirst5 is called on it.
///       In such a case, an error should be returned, where right now, the Omit-length is
///       being clamped to the base_word-length.
pub fn transformation(id: usize, base_word: &[u8]) -> Vec<u8> {
	match id {
		  0 => Vec::from(base_word),
		  1 => [Vec::from(base_word), vec![0x20]].concat(),
		  2 => [vec![0x20], Vec::from(base_word), vec![0x20]].concat(),
		  3 => Vec::from(&base_word[min(1, base_word.len() - 1)..]),
		  4 => [uppercase_first(base_word), vec![0x20]].concat(),
		  5 => [Vec::from(base_word), vec![0x20, 0x74, 0x68, 0x65, 0x20]].concat(),
		  6 => [vec![0x20], Vec::from(base_word)].concat(),
		  7 => [vec![0x73, 0x20], Vec::from(base_word), vec![0x20]].concat(),
		  8 => [Vec::from(base_word), vec![0x20, 0x6f, 0x66, 0x20]].concat(),
		  9 => uppercase_first(base_word),
		 10 => [Vec::from(base_word), vec![0x20, 0x61, 0x6e, 0x64, 0x20]].concat(),
		 11 => Vec::from(&base_word[min(2, base_word.len() - 1)..]),
		 12 => Vec::from(&base_word[..max(1, base_word.len()) - 1]),
		 13 => [vec![0x2c, 0x20], Vec::from(base_word), vec![0x20]].concat(),
		 14 => [Vec::from(base_word), vec![0x2c, 0x20]].concat(),
		 15 => [vec![0x20], uppercase_first(base_word), vec![0x20]].concat(),
		 16 => [Vec::from(base_word), vec![0x20, 0x69, 0x6e, 0x20]].concat(),
		 17 => [Vec::from(base_word), vec![0x20, 0x74, 0x6f, 0x20]].concat(),
		 18 => [vec![0x65, 0x20], Vec::from(base_word), vec![0x20]].concat(),
		 19 => [Vec::from(base_word), vec![0x22]].concat(),
		 20 => [Vec::from(base_word), vec![0x2e]].concat(),
		 21 => [Vec::from(base_word), vec![0x22, 0x3e]].concat(),
		 22 => [Vec::from(base_word), vec![0x0a]].concat(),
		 23 => Vec::from(&base_word[..max(3, base_word.len()) - 3]),
		 24 => [Vec::from(base_word), vec![0x5d]].concat(),
		 25 => [Vec::from(base_word), vec![0x20, 0x66, 0x6f, 0x72, 0x20]].concat(),
		 26 => Vec::from(&base_word[min(3, base_word.len() - 1)..]),<commit_msg>MOD: removed @TODO note, as the current behavior conforms with the spec after all<commit_after>use std::cmp::{ max, min };

fn uppercase_all(base_word: &[u8]) -> Vec<u8> {
	let l = base_word.len();
	let mut v = Vec::with_capacity(l);
	let mut i = 0;

	while i < l {
		match base_word[i] {
			0...96|123...191 => {
				v.push(base_word[i]);
				i += 1;
			},
			97...122 => {
				v.push(base_word[i] ^ 32);
				i += 1;
			},
			192...223 => {
				v.push(base_word[i]);
				if i + 1 < l {
					v.push(base_word[i + 1] ^ 32);
				}
				i += 2;
			},
			224...255 => {
				v.push(base_word[i]);
				if i + 1 < l {
					v.push(base_word[i + 1]);
				}
				if i + 2 < l {
					v.push(base_word[i + 2] ^ 5);
				}
				i = i + 3;
			},
			_ => unreachable!(), // confirmed unreachable, all byte values covered
		}
	}

	v
}

fn uppercase_first(base_word: &[u8]) -> Vec<u8> {
	let l = base_word.len();

	if l == 0 {
		return Vec::new();
	}

	let mut v = Vec::with_capacity(l);
	let i;

	match base_word[0] {
		1...96|123...191 => {
			v.push(base_word[0]);
			i = 1;
		},
		97...122 => {
			v.push(base_word[0] ^ 32);
			i = 1;
		},
		192...223 => {
			v.push(base_word[0]);
			if 1 < l {
				v.push(base_word[1] ^ 32);
			}
			i = 2;
		},
		224...255 => {
			v.push(base_word[0]);
			if 1 < l {
				v.push(base_word[1]);
			}
			if 2 < l {
				v.push(base_word[2] ^ 5);
			}
			i = 3;
		},
		_ => unreachable!(), // confirmed unreachable, all byte values covered
	}

	[v, Vec::from(&base_word[i..])].concat()
}

pub fn transformation(id: usize, base_word: &[u8]) -> Vec<u8> {
	match id {
		  0 => Vec::from(base_word),
		  1 => [Vec::from(base_word), vec![0x20]].concat(),
		  2 => [vec![0x20], Vec::from(base_word), vec![0x20]].concat(),
		  3 => Vec::from(&base_word[min(1, base_word.len() - 1)..]),
		  4 => [uppercase_first(base_word), vec![0x20]].concat(),
		  5 => [Vec::from(base_word), vec![0x20, 0x74, 0x68, 0x65, 0x20]].concat(),
		  6 => [vec![0x20], Vec::from(base_word)].concat(),
		  7 => [vec![0x73, 0x20], Vec::from(base_word), vec![0x20]].concat(),
		  8 => [Vec::from(base_word), vec![0x20, 0x6f, 0x66, 0x20]].concat(),
		  9 => uppercase_first(base_word),
		 10 => [Vec::from(base_word), vec![0x20, 0x61, 0x6e, 0x64, 0x20]].concat(),
		 11 => Vec::from(&base_word[min(2, base_word.len() - 1)..]),
		 12 => Vec::from(&base_word[..max(1, base_word.len()) - 1]),
		 13 => [vec![0x2c, 0x20], Vec::from(base_word), vec![0x20]].concat(),
		 14 => [Vec::from(base_word), vec![0x2c, 0x20]].concat(),
		 15 => [vec![0x20], uppercase_first(base_word), vec![0x20]].concat(),
		 16 => [Vec::from(base_word), vec![0x20, 0x69, 0x6e, 0x20]].concat(),
		 17 => [Vec::from(base_word), vec![0x20, 0x74, 0x6f, 0x20]].concat(),
		 18 => [vec![0x65, 0x20], Vec::from(base_word), vec![0x20]].concat(),
		 19 => [Vec::from(base_word), vec![0x22]].concat(),
		 20 => [Vec::from(base_word), vec![0x2e]].concat(),
		 21 => [Vec::from(base_word), vec![0x22, 0x3e]].concat(),
		 22 => [Vec::from(base_word), vec![0x0a]].concat(),
		 23 => Vec::from(&base_word[..max(3, base_word.len()) - 3]),
		 24 => [Vec::from(base_word), vec![0x5d]].concat(),
		 25 => [Vec::from(base_word), vec![0x20, 0x66, 0x6f, 0x72, 0x20]].concat(),
		 26 => Vec::from(&base_word[min(3, base_word.len() - 1)..]),
{'fraction_non_alphanumeric': 0.24088022448678187, 'fraction_numerical': 0.08625018461084034, 'mean_word_length': 3.1571516267648865, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 89, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5417459', 'n_tokens_mistral': 3596, 'n_tokens_neox': 2900, 'n_words': 565}