<commit_before>static inline void
enc_loop_generic_64_inner (const uint8_t **s, uint8_t **o)
{
	uint64_t src;

	// Load input:
	memcpy(&src, *s, sizeof (src));

	// Reorder to 64-bit big-endian, if not already in that format. The
	// workset must be in big-endian, otherwise the shifted bits do not
	// carry over properly among adjacent bytes:
	src = BASE64_HTOBE64(src);

	// Shift input by 6 bytes each round and mask in only the lower 6 bits;
	// look up the character in the Base64 encoding table and write it to
	// the output location:
	*(*o)++ = base64_table_enc_6bit[(src >> 58) & 0x3F];
	*(*o)++ = base64_table_enc_6bit[(src >> 52) & 0x3F];
	*(*o)++ = base64_table_enc_6bit[(src >> 46) & 0x3F];
	*(*o)++ = base64_table_enc_6bit[(src >> 40) & 0x3F];
	*(*o)++ = base64_table_enc_6bit[(src >> 34) & 0x3F];
	*(*o)++ = base64_table_enc_6bit[(src >> 28) & 0x3F];
	*(*o)++ = base64_table_enc_6bit[(src >> 22) & 0x3F];
	*(*o)++ = base64_table_enc_6bit[(src >> 16) & 0x3F];

	*s += 6;
}
<commit_msg>Generic64: enc: use 12-bit lookup table<commit_after>static inline void
enc_loop_generic_64_inner (const uint8_t **s, uint8_t **o)
{
	uint64_t src;

	// Load input:
	memcpy(&src, *s, sizeof (src));

	// Reorder to 64-bit big-endian, if not already in that format. The
	// workset must be in big-endian, otherwise the shifted bits do not
	// carry over properly among adjacent bytes:
	src = BASE64_HTOBE64(src);

	// Four indices for the 12-bit lookup table:
	const size_t index0 = (src >> 52) & 0xFFFU;
	const size_t index1 = (src >> 40) & 0xFFFU;
	const size_t index2 = (src >> 28) & 0xFFFU;
	const size_t index3 = (src >> 16) & 0xFFFU;

	// Table lookup and store:
	memcpy(*o + 0, base64_table_enc_12bit + index0, 2);
	memcpy(*o + 2, base64_table_enc_12bit + index1, 2);
	memcpy(*o + 4, base64_table_enc_12bit + index2, 2);
	memcpy(*o + 6, base64_table_enc_12bit + index3, 2);

	*s += 6;
	*o += 8;
}

{'fraction_non_alphanumeric': 0.15457413249211358, 'fraction_numerical': 0.07202944269190326, 'mean_word_length': 4.101876675603217, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2950699', 'n_tokens_mistral': 855, 'n_tokens_neox': 749, 'n_words': 223}