<commit_before>#include "stdafx.h"
#include "CppUnitTest.h"
#include "Interpret/SmartView/SmartView.h"
#include "MFCMAPI.h"
#include "resource.h"

using namespace Microsoft::VisualStudio::CppUnitTestFramework;

namespace SmartViewTest
{
	TEST_CLASS(SmartViewTest)
	{
	private:
		struct SmartViewTestResource
		{
			__ParsingTypeEnum structType;
			bool parseAll;
			DWORD hex;
			DWORD expected;
		};

		struct SmartViewTestData
		{
			__ParsingTypeEnum structType;
			bool parseAll;
			wstring testName;
			vector<BYTE> hex;
			wstring expected;
		};

		static const bool parse_all = false;
		static const bool assert_on_failure = true;
		static const bool limit_output = true;

		// Assert::AreEqual doesn't do full logging, so we roll our own
		void AreEqualEx(const wstring& expected, const wstring& actual, const wchar_t* message = nullptr, const __LineInfo* pLineInfo = nullptr) const
		{
			if (expected != actual)
			{
				if (message != nullptr)
				{
					Logger::WriteMessage(format(L"Test: %ws\n", message).c_str());
				}

				Logger::WriteMessage(L"Diff:\n");

				auto splitExpected = split(expected, L'\n');
				auto splitActual = split(actual, L'\n');
				auto errorCount = 0;
				for (size_t line = 0; line < splitExpected.size() && line < splitActual.size() && (errorCount < 4 || !limit_output); line++)
				{
					if (splitExpected[line] != splitActual[line])
					{
						errorCount++;
						Logger::WriteMessage(format(L"[%d]\n\"%ws\"\n\"%ws\"\n", line + 1, splitExpected[line].c_str(), splitActual[line].c_str()).c_str());
						auto lineErrorCount = 0;
						for (size_t ch = 0; ch < splitExpected[line].size() && ch < splitActual[line].size() && (lineErrorCount < 10 || !limit_output); ch++)
						{
							wchar_t expectedChar = splitExpected[line][ch];
							wchar_t actualChar = splitActual[line][ch];
							if (expectedChar != actualChar)
							{
								lineErrorCount++;
								Logger::WriteMessage(format(L"[%d]: %X (%wc) != %X (%wc)\n", ch + 1, expectedChar, expectedChar, actualChar, actualChar).c_str());
							}
						}
					}
				}

				Logger::WriteMessage(L"\n");
				Logger::WriteMessage(format(L"Expected:\n\"%ws\"\n\n", expected.c_str()).c_str());
				Logger::WriteMessage(format(L"Actual:\n\"%ws\"", actual.c_str()).c_str());

				if (assert_on_failure)
				{
					Assert::Fail(ToString(message).c_str(), pLineInfo);
				}
			}
		}

		void test(vector<SmartViewTestData> testData) const
		{
			for (auto data : testData)
			{
				auto actual = InterpretBinaryAsString({ static_cast<ULONG>(data.hex.size()), data.hex.data() }, data.structType, nullptr);
				AreEqualEx(data.expected, actual, data.testName.c_str());

				if (data.parseAll)
				{
					for (ULONG i = IDS_STNOPARSING; i < IDS_STEND; i++)
					{
						const auto structType = static_cast<__ParsingTypeEnum>(i);
						actual = InterpretBinaryAsString({ static_cast<ULONG>(data.hex.size()), data.hex.data() }, structType, nullptr);
						//Logger::WriteMessage(format(L"Testing %ws\n", AddInStructTypeToString(structType).c_str()).c_str());
						Assert::IsTrue(actual.length() != 0);
					}
				}
			}
		}

		// Resource files saved in unicode have a byte order mark of 0xfffe
		// We load these in and strip the BOM.
		// Otherwise we load as ansi and convert to unicode
		static wstring loadfile(const HMODULE handle, const int name)
		{
			const auto rc = ::FindResource(handle, MAKEINTRESOURCE(name), MAKEINTRESOURCE(TEXTFILE));
			const auto rcData = LoadResource(handle, rc);
			const auto cb = SizeofResource(handle, rc);
			const auto bytes = LockResource(rcData);<commit_msg>Test all inputs against all parsers looking for exceptions<commit_after>#include "stdafx.h"
#include "CppUnitTest.h"
#include "Interpret/SmartView/SmartView.h"
#include "MFCMAPI.h"
#include "resource.h"

using namespace Microsoft::VisualStudio::CppUnitTestFramework;

namespace SmartViewTest
{
	TEST_CLASS(SmartViewTest)
	{
	private:
		struct SmartViewTestResource
		{
			__ParsingTypeEnum structType;
			bool parseAll;
			DWORD hex;
			DWORD expected;
		};

		struct SmartViewTestData
		{
			__ParsingTypeEnum structType;
			bool parseAll;
			wstring testName;
			vector<BYTE> hex;
			wstring expected;
		};

		static const bool parse_all = true;
		static const bool assert_on_failure = true;
		static const bool limit_output = true;

		// Assert::AreEqual doesn't do full logging, so we roll our own
		void AreEqualEx(const wstring& expected, const wstring& actual, const wchar_t* message = nullptr, const __LineInfo* pLineInfo = nullptr) const
		{
			if (expected != actual)
			{
				if (message != nullptr)
				{
					Logger::WriteMessage(format(L"Test: %ws\n", message).c_str());
				}

				Logger::WriteMessage(L"Diff:\n");

				auto splitExpected = split(expected, L'\n');
				auto splitActual = split(actual, L'\n');
				auto errorCount = 0;
				for (size_t line = 0; line < splitExpected.size() && line < splitActual.size() && (errorCount < 4 || !limit_output); line++)
				{
					if (splitExpected[line] != splitActual[line])
					{
						errorCount++;
						Logger::WriteMessage(format(L"[%d]\n\"%ws\"\n\"%ws\"\n", line + 1, splitExpected[line].c_str(), splitActual[line].c_str()).c_str());
						auto lineErrorCount = 0;
						for (size_t ch = 0; ch < splitExpected[line].size() && ch < splitActual[line].size() && (lineErrorCount < 10 || !limit_output); ch++)
						{
							wchar_t expectedChar = splitExpected[line][ch];
							wchar_t actualChar = splitActual[line][ch];
							if (expectedChar != actualChar)
							{
								lineErrorCount++;
								Logger::WriteMessage(format(L"[%d]: %X (%wc) != %X (%wc)\n", ch + 1, expectedChar, expectedChar, actualChar, actualChar).c_str());
							}
						}
					}
				}

				Logger::WriteMessage(L"\n");
				Logger::WriteMessage(format(L"Expected:\n\"%ws\"\n\n", expected.c_str()).c_str());
				Logger::WriteMessage(format(L"Actual:\n\"%ws\"", actual.c_str()).c_str());

				if (assert_on_failure)
				{
					Assert::Fail(ToString(message).c_str(), pLineInfo);
				}
			}
		}

		void test(vector<SmartViewTestData> testData) const
		{
			for (auto data : testData)
			{
				auto actual = InterpretBinaryAsString({ static_cast<ULONG>(data.hex.size()), data.hex.data() }, data.structType, nullptr);
				AreEqualEx(data.expected, actual, data.testName.c_str());

				if (data.parseAll)
				{
					for (ULONG iStruct = IDS_STNOPARSING; iStruct < IDS_STEND; iStruct++)
					{
						const auto structType = static_cast<__ParsingTypeEnum>(iStruct);
						try
						{
							actual = InterpretBinaryAsString({ static_cast<ULONG>(data.hex.size()), data.hex.data() }, structType, nullptr);
						}
						catch (int exception)
						{
							Logger::WriteMessage(format(L"Testing %ws failed at %ws with error 0x%08X\n", data.testName.c_str(), AddInStructTypeToString(structType).c_str(), exception).c_str());
							Assert::Fail();
						}
					}
				}
			}
		}

		// Resource files saved in unicode have a byte order mark of 0xfffe
		// We load these in and strip the BOM.
		// Otherwise we load as ansi and convert to unicode
		static wstring loadfile(const HMODULE handle, const int name)
		{
			const auto rc = ::FindResource(handle, MAKEINTRESOURCE(name), MAKEINTRESOURCE(TEXTFILE));
			const auto rcData = LoadResource(handle, rc);
			const auto cb = SizeofResource(handle, rc);
			const auto bytes = LockResource(rcData);
{'fraction_non_alphanumeric': 0.14367114367114367, 'fraction_numerical': 0.003256003256003256, 'mean_word_length': 3.639395846444305, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2819753', 'n_tokens_mistral': 2986, 'n_tokens_neox': 2388, 'n_words': 550}