<commit_before>#pragma once

#include <string>
#include <vector>
#include <stdio.h>
#include <assert.h>
#include <Windows.h>

namespace Utility
{
	class String
	{
	public:
		//http://www.joinc.co.kr/modules/moniwiki/wiki.php/Site/C++/Documents/C++ProgramingHowToPerPage/standardstring.html
		static std::vector<std::string> Tokenize(const std::string& str, const std::string& delimiters = " ")
		{
			std::vector<std::string> tokens;

			//  ù ڰ   
			std::string::size_type lastPos = str.find_first_not_of(delimiters, 0);
			// ڰ ƴ ù ڸ ã´
			std::string::size_type pos     = str.find_first_of(delimiters, lastPos);

			while (std::string::npos != pos || std::string::npos != lastPos)
			{
				// token ã vector ߰Ѵ
				tokens.push_back(str.substr(lastPos, pos - lastPos));
				// ڸ پѴ´.  "not_of" ϶
				lastPos = str.find_first_not_of(delimiters, pos);
				//  ڰ ƴ ڸ ã´
				pos = str.find_first_of(delimiters, lastPos);
			}

			return tokens;
		}

		static bool ParseDirectory(const std::string& path, std::string& outFolderPath, std::string& outFileName, std::string& outFileFormat)
		{
			return ParseDirectory(path, &outFolderPath, &outFileName, &outFileFormat);
		}

		static bool ParseDirectory(const std::string& path, std::string* outFolderPath, std::string* outFileName, std::string* outFileFormat)
		{
			auto ParseNameAndFormat = [&](const std::string& fileNameWithFormat)
			{
				unsigned int formatPos = fileNameWithFormat.find('.');
				if( formatPos == -1 )
					return false;

				if(outFileFormat)
					*outFileFormat = &fileNameWithFormat.c_str()[formatPos+1];

				if(outFileName)
					*outFileName = fileNameWithFormat.substr(0, formatPos); 

				return true;
			};

			unsigned int fileNameStartPos = path.rfind('/');
			if( fileNameStartPos == -1 )
			{
				fileNameStartPos = path.rfind('\\');
				if(fileNameStartPos == -1)
					return ParseNameAndFormat(path);
			}

			if(outFolderPath)
				*outFolderPath = path.substr(0, fileNameStartPos+1);

			return ParseNameAndFormat( &path.c_str()[fileNameStartPos+1] );
		}

		static uint MakeKey(const std::vector<std::string>& keys)
		{
			std::string result = "";
			for (auto& iter : keys)
				result += iter;

			return std::hash<std::string>()(result);
		}

	};

	template <unsigned int CheckValue, unsigned int searchPos = sizeof(int) * 8 - 1>
	class Compute_2_Exp
	{
	public:
		template <int n, int pn = searchPos + 1>
		struct CheckBit
		{
			constexpr static unsigned int value =
				CheckBit<uint(((1 << n) & CheckValue) == 0) * n - 1, n>::value;
		};

		template <int pn>
		struct CheckBit<-1, pn>
		{
			constexpr static unsigned int value = pn;
		};

		constexpr static unsigned int value = CheckBit<searchPos>::value;
	};
}<commit_msg>코드 정리<commit_after>#pragma once

#include <string>
#include <vector>
#include <stdio.h>
#include <assert.h>
#include <Windows.h>

namespace Utility
{
	class String
	{
	public:
		//http://www.joinc.co.kr/modules/moniwiki/wiki.php/Site/C++/Documents/C++ProgramingHowToPerPage/standardstring.html
		static std::vector<std::string> Tokenize(const std::string& str, const std::string& delimiters = " ")
		{
			std::vector<std::string> tokens;

			//  ù ڰ   
			std::string::size_type lastPos = str.find_first_not_of(delimiters, 0);
			// ڰ ƴ ù ڸ ã´
			std::string::size_type pos     = str.find_first_of(delimiters, lastPos);

			while (std::string::npos != pos || std::string::npos != lastPos)
			{
				// token ã vector ߰Ѵ
				tokens.push_back(str.substr(lastPos, pos - lastPos));
				// ڸ پѴ´.  "not_of" ϶
				lastPos = str.find_first_not_of(delimiters, pos);
				//  ڰ ƴ ڸ ã´
				pos = str.find_first_of(delimiters, lastPos);
			}

			return tokens;
		}

		static bool ParseDirectory(const std::string& path, std::string& outFolderPath, std::string& outFileName, std::string& outFileFormat)
		{
			return ParseDirectory(path, &outFolderPath, &outFileName, &outFileFormat);
		}

		static bool ParseDirectory(const std::string& path, std::string* outFolderPath, std::string* outFileName, std::string* outFileFormat)
		{
			auto ParseNameAndFormat = [&](const std::string& fileNameWithFormat)
			{
				unsigned int formatPos = fileNameWithFormat.find('.');
				if( formatPos == -1 )
					return false;

				if(outFileFormat)
					*outFileFormat = &fileNameWithFormat.c_str()[formatPos+1];

				if(outFileName)
					*outFileName = fileNameWithFormat.substr(0, formatPos); 

				return true;
			};

			unsigned int fileNameStartPos = path.rfind('/');
			if( fileNameStartPos == -1 )
			{
				fileNameStartPos = path.rfind('\\');
				if(fileNameStartPos == -1)
					return ParseNameAndFormat(path);
			}

			if(outFolderPath)
				*outFolderPath = path.substr(0, fileNameStartPos+1);

			return ParseNameAndFormat( &path.c_str()[fileNameStartPos+1] );
		}

		static uint MakeKey(const std::vector<std::string>& keys)
		{
			std::string result = "";
			for (auto& iter : keys)
				result += iter;

			return std::hash<std::string>()(result);
		}

	};

	template <unsigned int CheckValue, unsigned int searchPos = sizeof(int) * 8 - 1>
	class Compute_2_Exp
	{
	private:
		template <int n, int pn = searchPos + 1>
		struct CheckBit
		{
			constexpr static unsigned int value =
				CheckBit<uint(((1 << n) & CheckValue) == 0) * n - 1, n>::value;
		};

		template <int pn>
		struct CheckBit<-1, pn>
		{
			constexpr static unsigned int value = pn;
		};

	public:
		constexpr static unsigned int value = CheckBit<searchPos>::value;
	};
}
{'fraction_non_alphanumeric': 0.14503676470588237, 'fraction_numerical': 0.00625, 'mean_word_length': 3.7478184991273995, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '395325', 'n_tokens_mistral': 2219, 'n_tokens_neox': 1918, 'n_words': 418}