<commit_before>#include "tests-base.hpp"

#include "utf8rewind.h"

#include "helpers-seeking.hpp"

TEST(Utf8SeekEnd, OneByteSingle)
{
	const char* t = "Z";

	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 1, SEEK_END);
}

TEST(Utf8SeekEnd, OneByteSingleFirst)
{
	const char* t = "\0";

	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 1, SEEK_END);
}

TEST(Utf8SeekEnd, OneByteSingleLast)
{
	const char* t = "\x7F";

	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 1, SEEK_END);
}

TEST(Utf8SeekEnd, OneByteSingleInvalidContinuationByteFirst)
{
	const char* t = "\x80";

	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 1, SEEK_END);
}

TEST(Utf8SeekEnd, OneByteSingleInvalidContinuationByteLast)
{
	const char* t = "\xBF";

	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 1, SEEK_END);
}

TEST(Utf8SeekEnd, OneByteSingleOverlongOneByte)
{
	const char* t = "\x26\x9C";

	EXPECT_SEEKEQ(t, 1, strlen(t), 0, 1, SEEK_END);
	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 2, SEEK_END);
}

TEST(Utf8SeekEnd, OneByteSingleOverlongTwoBytes)
{
	const char* t = "\x51\xA1\xB0";

	EXPECT_SEEKEQ(t, 2, strlen(t), 0, 1, SEEK_END);
	EXPECT_SEEKEQ(t, 1, strlen(t), 0, 2, SEEK_END);
	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 3, SEEK_END);
}

TEST(Utf8SeekEnd, OneByteSingleOverlongThreeBytes)
{
	const char* t = "\x24\x99\xA2\x81";

	EXPECT_SEEKEQ(t, 3, strlen(t), 0, 1, SEEK_END);
	EXPECT_SEEKEQ(t, 2, strlen(t), 0, 2, SEEK_END);
	EXPECT_SEEKEQ(t, 1, strlen(t), 0, 3, SEEK_END);
	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 4, SEEK_END);
}

TEST(Utf8SeekEnd, OneByteSingleOverlongFourBytes)
{
	const char* t = "\x3B\x9A\x80\x92\x86";

	EXPECT_SEEKEQ(t, 4, strlen(t), 0, 1, SEEK_END);
	EXPECT_SEEKEQ(t, 3, strlen(t), 0, 2, SEEK_END);
	EXPECT_SEEKEQ(t, 2, strlen(t), 0, 3, SEEK_END);
	EXPECT_SEEKEQ(t, 1, strlen(t), 0, 4, SEEK_END);
	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 5, SEEK_END);
}

TEST(Utf8SeekEnd, OneByteSingleOverlongFiveBytes)
{
	const char* t = "\x45\x86\x91\xA2\x85\x92";

	EXPECT_SEEKEQ(t, 5, strlen(t), 0, 1, SEEK_END);
	EXPECT_SEEKEQ(t, 4, strlen(t), 0, 2, SEEK_END);
	EXPECT_SEEKEQ(t, 3, strlen(t), 0, 3, SEEK_END);
	EXPECT_SEEKEQ(t, 2, strlen(t), 0, 4, SEEK_END);
	EXPECT_SEEKEQ(t, 1, strlen(t), 0, 5, SEEK_END);
	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 6, SEEK_END);
}

TEST(Utf8SeekEnd, OneByteSingleOverlongSixBytes)
{
	const char* t = "\x5F\x92\xA9\x92\xA5\x82\x99";

	EXPECT_SEEKEQ(t, 6, strlen(t), 0, 1, SEEK_END);
	EXPECT_SEEKEQ(t, 5, strlen(t), 0, 2, SEEK_END);
	EXPECT_SEEKEQ(t, 4, strlen(t), 0, 3, SEEK_END);
	EXPECT_SEEKEQ(t, 3, strlen(t), 0, 4, SEEK_END);
	EXPECT_SEEKEQ(t, 2, strlen(t), 0, 5, SEEK_END);
	EXPECT_SEEKEQ(t, 1, strlen(t), 0, 6, SEEK_END);
	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 7, SEEK_END);
}

TEST(Utf8SeekEnd, OneByteMultiple)
{
	const char* t = "Farmer";

	EXPECT_SEEKEQ(t, 5, strlen(t), 0, 1, SEEK_END);
	EXPECT_SEEKEQ(t, 4, strlen(t), 0, 2, SEEK_END);
	EXPECT_SEEKEQ(t, 3, strlen(t), 0, 3, SEEK_END);
	EXPECT_SEEKEQ(t, 2, strlen(t), 0, 4, SEEK_END);
	EXPECT_SEEKEQ(t, 1, strlen(t), 0, 5, SEEK_END);
	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 6, SEEK_END);
}

TEST(Utf8SeekEnd, OneByteMultipleContinuationBytes)
{
	const char* t = "\x95\xA5\x92\xB2\x90";

	EXPECT_SEEKEQ(t, 4, strlen(t), 0, 1, SEEK_END);
	EXPECT_SEEKEQ(t, 3, strlen(t), 0, 2, SEEK_END);
	EXPECT_SEEKEQ(t, 2, strlen(t), 0, 3, SEEK_END);
	EXPECT_SEEKEQ(t, 1, strlen(t), 0, 4, SEEK_END);
	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 5, SEEK_END);
}

TEST(Utf8SeekEnd, OneByteMultipleOverlong)
{
	const char* t = "\x86\xB4\xA2"
					"\x72\xB1\x8D\x87\x95"
					"\x41\x90\x84";

	EXPECT_SEEKEQ(t, 10, strlen(t), 0, 1, SEEK_END);
	EXPECT_SEEKEQ(t, 9, strlen(t), 0, 2, SEEK_END);
	EXPECT_SEEKEQ(t, 8, strlen(t), 0, 3, SEEK_END);
	EXPECT_SEEKEQ(t, 7, strlen(t), 0, 4, SEEK_END);
	EXPECT_SEEKEQ(t, 6, strlen(t), 0, 5, SEEK_END);
	EXPECT_SEEKEQ(t, 5, strlen(t), 0, 6, SEEK_END);
	EXPECT_SEEKEQ(t, 4, strlen(t), 0, 7, SEEK_END);
	EXPECT_SEEKEQ(t, 3, strlen(t), 0, 8, SEEK_END);
	EXPECT_SEEKEQ(t, 2, strlen(t), 0, 9, SEEK_END);
	EXPECT_SEEKEQ(t, 1, strlen(t), 0, 10, SEEK_END);
	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 11, SEEK_END);
}

TEST(Utf8SeekEnd, Valid)
{
	const char* t = "\xF0\x90\x92\x80\xF0\x90\x92\x80\xF0\x90\x92\x80\xF0\x90\x92\x80\xF0\x90\x92\x80";

	const char* r = utf8seek(t, t, 4, SEEK_END);

	EXPECT_EQ(t + 4, r);
	EXPECT_STREQ("\xF0\x90\x92\x80\xF0\x90\x92\x80\xF0\x90\x92\x80\xF0\x90\x92\x80", r);

	unicode_t o = 0;
	EXPECT_EQ(4, utf8toutf32(r, strlen(r), &o, sizeof(o), nullptr));
	EXPECT_EQ(0x10480, o);
}

TEST(Utf8SeekEnd, Offset)
{
	const char* t = "\xF0\x90\x92\x80\xF0\x90\x92\x80\xF0\x90\x92\x80\xF0\x90\x92\x80\xF0\x90\x92\x80";

	const char* r = utf8seek(t + 12, t, 2, SEEK_END);

	EXPECT_EQ(t + 12, r);
	EXPECT_STREQ("\xF0\x90\x92\x80\xF0\x90\x92\x80", r);

	unicode_t o = 0;
	EXPECT_EQ(4, utf8toutf32(r, strlen(r), &o, sizeof(o), nullptr));
	EXPECT_EQ(0x10480, o);<commit_msg>Refactor tests for two byte sequences.<commit_after>#include "tests-base.hpp"

#include "utf8rewind.h"

#include "helpers-seeking.hpp"

TEST(Utf8SeekEnd, OneByteSingle)
{
	const char* t = "Z";

	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 1, SEEK_END);
}

TEST(Utf8SeekEnd, OneByteSingleFirst)
{
	const char* t = "\0";

	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 1, SEEK_END);
}

TEST(Utf8SeekEnd, OneByteSingleLast)
{
	const char* t = "\x7F";

	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 1, SEEK_END);
}

TEST(Utf8SeekEnd, OneByteSingleInvalidContinuationByteFirst)
{
	const char* t = "\x80";

	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 1, SEEK_END);
}

TEST(Utf8SeekEnd, OneByteSingleInvalidContinuationByteLast)
{
	const char* t = "\xBF";

	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 1, SEEK_END);
}

TEST(Utf8SeekEnd, OneByteSingleOverlongOneByte)
{
	const char* t = "\x26\x9C";

	EXPECT_SEEKEQ(t, 1, strlen(t), 0, 1, SEEK_END);
	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 2, SEEK_END);
}

TEST(Utf8SeekEnd, OneByteSingleOverlongTwoBytes)
{
	const char* t = "\x51\xA1\xB0";

	EXPECT_SEEKEQ(t, 2, strlen(t), 0, 1, SEEK_END);
	EXPECT_SEEKEQ(t, 1, strlen(t), 0, 2, SEEK_END);
	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 3, SEEK_END);
}

TEST(Utf8SeekEnd, OneByteSingleOverlongThreeBytes)
{
	const char* t = "\x24\x99\xA2\x81";

	EXPECT_SEEKEQ(t, 3, strlen(t), 0, 1, SEEK_END);
	EXPECT_SEEKEQ(t, 2, strlen(t), 0, 2, SEEK_END);
	EXPECT_SEEKEQ(t, 1, strlen(t), 0, 3, SEEK_END);
	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 4, SEEK_END);
}

TEST(Utf8SeekEnd, OneByteSingleOverlongFourBytes)
{
	const char* t = "\x3B\x9A\x80\x92\x86";

	EXPECT_SEEKEQ(t, 4, strlen(t), 0, 1, SEEK_END);
	EXPECT_SEEKEQ(t, 3, strlen(t), 0, 2, SEEK_END);
	EXPECT_SEEKEQ(t, 2, strlen(t), 0, 3, SEEK_END);
	EXPECT_SEEKEQ(t, 1, strlen(t), 0, 4, SEEK_END);
	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 5, SEEK_END);
}

TEST(Utf8SeekEnd, OneByteSingleOverlongFiveBytes)
{
	const char* t = "\x45\x86\x91\xA2\x85\x92";

	EXPECT_SEEKEQ(t, 5, strlen(t), 0, 1, SEEK_END);
	EXPECT_SEEKEQ(t, 4, strlen(t), 0, 2, SEEK_END);
	EXPECT_SEEKEQ(t, 3, strlen(t), 0, 3, SEEK_END);
	EXPECT_SEEKEQ(t, 2, strlen(t), 0, 4, SEEK_END);
	EXPECT_SEEKEQ(t, 1, strlen(t), 0, 5, SEEK_END);
	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 6, SEEK_END);
}

TEST(Utf8SeekEnd, OneByteSingleOverlongSixBytes)
{
	const char* t = "\x5F\x92\xA9\x92\xA5\x82\x99";

	EXPECT_SEEKEQ(t, 6, strlen(t), 0, 1, SEEK_END);
	EXPECT_SEEKEQ(t, 5, strlen(t), 0, 2, SEEK_END);
	EXPECT_SEEKEQ(t, 4, strlen(t), 0, 3, SEEK_END);
	EXPECT_SEEKEQ(t, 3, strlen(t), 0, 4, SEEK_END);
	EXPECT_SEEKEQ(t, 2, strlen(t), 0, 5, SEEK_END);
	EXPECT_SEEKEQ(t, 1, strlen(t), 0, 6, SEEK_END);
	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 7, SEEK_END);
}

TEST(Utf8SeekEnd, OneByteMultiple)
{
	const char* t = "Farmer";

	EXPECT_SEEKEQ(t, 5, strlen(t), 0, 1, SEEK_END);
	EXPECT_SEEKEQ(t, 4, strlen(t), 0, 2, SEEK_END);
	EXPECT_SEEKEQ(t, 3, strlen(t), 0, 3, SEEK_END);
	EXPECT_SEEKEQ(t, 2, strlen(t), 0, 4, SEEK_END);
	EXPECT_SEEKEQ(t, 1, strlen(t), 0, 5, SEEK_END);
	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 6, SEEK_END);
}

TEST(Utf8SeekEnd, OneByteMultipleContinuationBytes)
{
	const char* t = "\x95\xA5\x92\xB2\x90";

	EXPECT_SEEKEQ(t, 4, strlen(t), 0, 1, SEEK_END);
	EXPECT_SEEKEQ(t, 3, strlen(t), 0, 2, SEEK_END);
	EXPECT_SEEKEQ(t, 2, strlen(t), 0, 3, SEEK_END);
	EXPECT_SEEKEQ(t, 1, strlen(t), 0, 4, SEEK_END);
	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 5, SEEK_END);
}

TEST(Utf8SeekEnd, OneByteMultipleOverlong)
{
	const char* t = "\x86\xB4\xA2"
					"\x72\xB1\x8D\x87\x95"
					"\x41\x90\x84";

	EXPECT_SEEKEQ(t, 10, strlen(t), 0, 1, SEEK_END);
	EXPECT_SEEKEQ(t, 9, strlen(t), 0, 2, SEEK_END);
	EXPECT_SEEKEQ(t, 8, strlen(t), 0, 3, SEEK_END);
	EXPECT_SEEKEQ(t, 7, strlen(t), 0, 4, SEEK_END);
	EXPECT_SEEKEQ(t, 6, strlen(t), 0, 5, SEEK_END);
	EXPECT_SEEKEQ(t, 5, strlen(t), 0, 6, SEEK_END);
	EXPECT_SEEKEQ(t, 4, strlen(t), 0, 7, SEEK_END);
	EXPECT_SEEKEQ(t, 3, strlen(t), 0, 8, SEEK_END);
	EXPECT_SEEKEQ(t, 2, strlen(t), 0, 9, SEEK_END);
	EXPECT_SEEKEQ(t, 1, strlen(t), 0, 10, SEEK_END);
	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 11, SEEK_END);
}

TEST(Utf8SeekEnd, TwoBytesSingle)
{
	const char* t = "\xC9\x86";

	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 1, SEEK_END);
}

TEST(Utf8SeekEnd, TwoBytesSingleFirst)
{
	const char* t = "\xC0\x80";

	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 1, SEEK_END);
}

TEST(Utf8SeekEnd, TwoBytesSingleLast)
{
	const char* t = "\xDF\xBF";

	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 1, SEEK_END);
}

TEST(Utf8SeekEnd, TwoBytesSingleInvalidContinuationFirstByteLower)
{
	const char* t = "\xCD\x24";

	EXPECT_SEEKEQ(t, 1, strlen(t), 0, 1, SEEK_END);
	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 2, SEEK_END);
}

TEST(Utf8SeekEnd, TwoBytesSingleInvalidContinuationFirstByteUpper)
{
	const char* t = "\xD8\xD5";

	EXPECT_SEEKEQ(t, 1, strlen(t), 0, 1, SEEK_END);
	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 2, SEEK_END);
}

TEST(Utf8SeekEnd, TwoBytesSingleMissingOneByte)
{
	const char* t = "\xCE";

	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 1, SEEK_END);
}

TEST(Utf8SeekEnd, TwoBytesSingleOverlongOneByte)
{
	const char* t = "\xC5\x92\xB2";

	EXPECT_SEEKEQ(t, 2, strlen(t), 0, 1, SEEK_END);
	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 2, SEEK_END);
}

TEST(Utf8SeekEnd, TwoBytesSingleOverlongTwoBytes)
{
	const char* t = "\xD4\x85\x92\x9A";

	EXPECT_SEEKEQ(t, 3, strlen(t), 0, 1, SEEK_END);
	EXPECT_SEEKEQ(t, 2, strlen(t), 0, 2, SEEK_END);
	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 3, SEEK_END);
}

TEST(Utf8SeekEnd, TwoBytesSingleOverlongThreeBytes)
{
	const char* t = "\xDD\x81\x9A\x9C\xBC";

	EXPECT_SEEKEQ(t, 4, strlen(t), 0, 1, SEEK_END);
	EXPECT_SEEKEQ(t, 3, strlen(t), 0, 2, SEEK_END);
	EXPECT_SEEKEQ(t, 2, strlen(t), 0, 3, SEEK_END);
	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 4, SEEK_END);
}

TEST(Utf8SeekEnd, TwoBytesSingleOverlongFourBytes)
{
	const char* t = "\xD1\x96\x92\xAC\x84\xB2";

	EXPECT_SEEKEQ(t, 5, strlen(t), 0, 1, SEEK_END);
	EXPECT_SEEKEQ(t, 4, strlen(t), 0, 2, SEEK_END);
	EXPECT_SEEKEQ(t, 3, strlen(t), 0, 3, SEEK_END);
	EXPECT_SEEKEQ(t, 2, strlen(t), 0, 4, SEEK_END);
	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 5, SEEK_END);
}

TEST(Utf8SeekEnd, TwoBytesSingleOverlongFiveBytes)
{
	const char* t = "\xDC\x92\x84\x92\x99\x9A\x83";

	EXPECT_SEEKEQ(t, 6, strlen(t), 0, 1, SEEK_END);
	EXPECT_SEEKEQ(t, 5, strlen(t), 0, 2, SEEK_END);
	EXPECT_SEEKEQ(t, 4, strlen(t), 0, 3, SEEK_END);
	EXPECT_SEEKEQ(t, 3, strlen(t), 0, 4, SEEK_END);
	EXPECT_SEEKEQ(t, 2, strlen(t), 0, 5, SEEK_END);
	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 6, SEEK_END);
}

TEST(Utf8SeekEnd, TwoBytesMultiple)
{
	const char* t = "\xD4\x9A\xD4\x9C\xD4\x86\xD4\x8A\xD4\xB6";

	EXPECT_SEEKEQ(t, 8, strlen(t), 0, 1, SEEK_END);
	EXPECT_SEEKEQ(t, 6, strlen(t), 0, 2, SEEK_END);
	EXPECT_SEEKEQ(t, 4, strlen(t), 0, 3, SEEK_END);
	EXPECT_SEEKEQ(t, 2, strlen(t), 0, 4, SEEK_END);
	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 5, SEEK_END);
}

TEST(Utf8SeekEnd, TwoBytesMultipleInvalid)
{
	const char* t = "\xC8\xD8\xC4\xC7\xDE";

	EXPECT_SEEKEQ(t, 4, strlen(t), 0, 1, SEEK_END);
	EXPECT_SEEKEQ(t, 3, strlen(t), 0, 2, SEEK_END);
	EXPECT_SEEKEQ(t, 2, strlen(t), 0, 3, SEEK_END);
	EXPECT_SEEKEQ(t, 1, strlen(t), 0, 4, SEEK_END);
	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 5, SEEK_END);
}

TEST(Utf8SeekEnd, TwoBytesMultipleOverlong)
{
	const char* t = "\xC9\x82\x99\x9A"
					"\xD1\x82\x92"
					"\xD4\x9A\x9A\xB2";

	EXPECT_SEEKEQ(t, 10, strlen(t), 0, 1, SEEK_END);
	EXPECT_SEEKEQ(t, 9, strlen(t), 0, 2, SEEK_END);
	EXPECT_SEEKEQ(t, 7, strlen(t), 0, 3, SEEK_END);
	EXPECT_SEEKEQ(t, 6, strlen(t), 0, 4, SEEK_END);
	EXPECT_SEEKEQ(t, 4, strlen(t), 0, 5, SEEK_END);
	EXPECT_SEEKEQ(t, 3, strlen(t), 0, 6, SEEK_END);
	EXPECT_SEEKEQ(t, 2, strlen(t), 0, 7, SEEK_END);
	EXPECT_SEEKEQ(t, 0, strlen(t), 0, 8, SEEK_END);
}

TEST(Utf8SeekEnd, Valid)
{
	const char* t = "\xF0\x90\x92\x80\xF0\x90\x92\x80\xF0\x90\x92\x80\xF0\x90\x92\x80\xF0\x90\x92\x80";

	const char* r = utf8seek(t, t, 4, SEEK_END);

	EXPECT_EQ(t + 4, r);
	EXPECT_STREQ("\xF0\x90\x92\x80\xF0\x90\x92\x80\xF0\x90\x92\x80\xF0\x90\x92\x80", r);

	unicode_t o = 0;
	EXPECT_EQ(4, utf8toutf32(r, strlen(r), &o, sizeof(o), nullptr));
	EXPECT_EQ(0x10480, o);
}

TEST(Utf8SeekEnd, Offset)
{
	const char* t = "\xF0\x90\x92\x80\xF0\x90\x92\x80\xF0\x90\x92\x80\xF0\x90\x92\x80\xF0\x90\x92\x80";

	const char* r = utf8seek(t + 12, t, 2, SEEK_END);

	EXPECT_EQ(t + 12, r);
	EXPECT_STREQ("\xF0\x90\x92\x80\xF0\x90\x92\x80", r);

	unicode_t o = 0;
	EXPECT_EQ(4, utf8toutf32(r, strlen(r), &o, sizeof(o), nullptr));
	EXPECT_EQ(0x10480, o);
{'fraction_non_alphanumeric': 0.18981940105158882, 'fraction_numerical': 0.0780309380477025, 'mean_word_length': 5.193487494100991, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2254426', 'n_tokens_mistral': 7502, 'n_tokens_neox': 6744, 'n_words': 1316}