<commit_before>
// StringUtils.cpp

// Implements the various string helper functions:

#include "Globals.h"

#if defined(ANDROID_NDK)
#include <ctype.h>
#endif

#ifdef _MSC_VER
	// Under MSVC, link to WinSock2 (needed by RawBEToUTF8's byteswapping)
	#pragma comment(lib, "ws2_32.lib")
#endif





AString & AppendVPrintf(AString & str, const char * format, va_list args)
{
	ASSERT(format != nullptr);
	
	char buffer[2048];
	int len;
	#ifdef va_copy
	va_list argsCopy;
	va_copy(argsCopy, args);
	#else
	#define argsCopy args
	#endif
	#ifdef _MSC_VER
	// MS CRT provides secure printf that doesn't behave like in the C99 standard
	if ((len = _vsnprintf_s(buffer, ARRAYCOUNT(buffer), _TRUNCATE, format, argsCopy)) != -1)
	#else  // _MSC_VER
	if ((len = vsnprintf(buffer, ARRAYCOUNT(buffer), format, argsCopy)) < static_cast<int>(ARRAYCOUNT(buffer)))
	#endif  // else _MSC_VER
	{
		// The result did fit into the static buffer
		#ifdef va_copy
		va_end(argsCopy);
		#endif
		str.append(buffer, static_cast<size_t>(len));
		return str;
	}
	#ifdef va_copy
	va_end(argsCopy);
	#endif
	
	// The result did not fit into the static buffer, use a dynamic buffer:
	#ifdef _MSC_VER
	// for MS CRT, we need to calculate the result length
	len = _vscprintf(format, args);
	if (len == -1)
	{
		return str;
	}
	#endif  // _MSC_VER
	
	// Allocate a buffer and printf into it:
	#ifdef va_copy
	va_copy(argsCopy, args);
	#endif
	std::vector<char> Buffer(static_cast<size_t>(len) + 1);
	#ifdef _MSC_VER
	vsprintf_s(&(Buffer.front()), Buffer.size(), format, argsCopy);
	#else  // _MSC_VER
	vsnprintf(&(Buffer.front()), Buffer.size(), format, argsCopy);
	#endif  // else _MSC_VER
	str.append(&(Buffer.front()), Buffer.size() - 1);
	#ifdef va_copy
	va_end(argsCopy);
	#endif
	return str;
}





AString & Printf(AString & str, const char * format, ...)
{
	str.clear();
	va_list args;
	va_start(args, format);
	std::string & retval = AppendVPrintf(str, format, args);
	va_end(args);
	return retval;
}





AString Printf(const char * format, ...)
{
	AString res;
	va_list args;
	va_start(args, format);
	AppendVPrintf(res, format, args);
	va_end(args);
	return res;
}





AString & AppendPrintf(AString & dst, const char * format, ...)
{
	va_list args;
	va_start(args, format);
	std::string & retval = AppendVPrintf(dst, format, args);
	va_end(args);
	return retval;
}





AStringVector StringSplit(const AString & str, const AString & delim)
{
	AStringVector results;
	size_t cutAt = 0;
	size_t Prev = 0;
	while ((cutAt = str.find_first_of(delim, Prev)) != str.npos)
	{
		results.push_back(str.substr(Prev, cutAt - Prev));
		Prev = cutAt + 1;
	}
	if (Prev < str.length())
	{
		results.push_back(str.substr(Prev));
	}
	return results;
}




AStringVector StringSplitWithQuotes(const AString & str, const AString & delim)
{
	AStringVector results;

	size_t cutAt = 0;
	size_t Prev = 0;
	size_t cutAtQuote = 0;

	while ((cutAt = str.find_first_of(delim, Prev)) != str.npos)
	{
		AString current = str.substr(Prev, cutAt - Prev);
		if ((current.front() == '"') || (current.front() == '\''))
		{
			Prev += 1;
			cutAtQuote = str.find_first_of(current.front(), Prev);
			if (cutAtQuote != str.npos)
			{
				current = str.substr(Prev, cutAtQuote - Prev);
				cutAt = cutAtQuote + 1;
			}
		}

		results.push_back(std::move(current));
		Prev = cutAt + 1;
	}<commit_msg>StringUtils: Fixed StringSplitWithQuotes().<commit_after>
// StringUtils.cpp

// Implements the various string helper functions:

#include "Globals.h"

#if defined(ANDROID_NDK)
#include <ctype.h>
#endif

#ifdef _MSC_VER
	// Under MSVC, link to WinSock2 (needed by RawBEToUTF8's byteswapping)
	#pragma comment(lib, "ws2_32.lib")
#endif





AString & AppendVPrintf(AString & str, const char * format, va_list args)
{
	ASSERT(format != nullptr);
	
	char buffer[2048];
	int len;
	#ifdef va_copy
	va_list argsCopy;
	va_copy(argsCopy, args);
	#else
	#define argsCopy args
	#endif
	#ifdef _MSC_VER
	// MS CRT provides secure printf that doesn't behave like in the C99 standard
	if ((len = _vsnprintf_s(buffer, ARRAYCOUNT(buffer), _TRUNCATE, format, argsCopy)) != -1)
	#else  // _MSC_VER
	if ((len = vsnprintf(buffer, ARRAYCOUNT(buffer), format, argsCopy)) < static_cast<int>(ARRAYCOUNT(buffer)))
	#endif  // else _MSC_VER
	{
		// The result did fit into the static buffer
		#ifdef va_copy
		va_end(argsCopy);
		#endif
		str.append(buffer, static_cast<size_t>(len));
		return str;
	}
	#ifdef va_copy
	va_end(argsCopy);
	#endif
	
	// The result did not fit into the static buffer, use a dynamic buffer:
	#ifdef _MSC_VER
	// for MS CRT, we need to calculate the result length
	len = _vscprintf(format, args);
	if (len == -1)
	{
		return str;
	}
	#endif  // _MSC_VER
	
	// Allocate a buffer and printf into it:
	#ifdef va_copy
	va_copy(argsCopy, args);
	#endif
	std::vector<char> Buffer(static_cast<size_t>(len) + 1);
	#ifdef _MSC_VER
	vsprintf_s(&(Buffer.front()), Buffer.size(), format, argsCopy);
	#else  // _MSC_VER
	vsnprintf(&(Buffer.front()), Buffer.size(), format, argsCopy);
	#endif  // else _MSC_VER
	str.append(&(Buffer.front()), Buffer.size() - 1);
	#ifdef va_copy
	va_end(argsCopy);
	#endif
	return str;
}





AString & Printf(AString & str, const char * format, ...)
{
	str.clear();
	va_list args;
	va_start(args, format);
	std::string & retval = AppendVPrintf(str, format, args);
	va_end(args);
	return retval;
}





AString Printf(const char * format, ...)
{
	AString res;
	va_list args;
	va_start(args, format);
	AppendVPrintf(res, format, args);
	va_end(args);
	return res;
}





AString & AppendPrintf(AString & dst, const char * format, ...)
{
	va_list args;
	va_start(args, format);
	std::string & retval = AppendVPrintf(dst, format, args);
	va_end(args);
	return retval;
}





AStringVector StringSplit(const AString & str, const AString & delim)
{
	AStringVector results;
	size_t cutAt = 0;
	size_t Prev = 0;
	while ((cutAt = str.find_first_of(delim, Prev)) != str.npos)
	{
		results.push_back(str.substr(Prev, cutAt - Prev));
		Prev = cutAt + 1;
	}
	if (Prev < str.length())
	{
		results.push_back(str.substr(Prev));
	}
	return results;
}




AStringVector StringSplitWithQuotes(const AString & str, const AString & delim)
{
	AStringVector results;

	size_t cutAt = 0;
	size_t Prev = 0;
	size_t cutAtQuote = 0;

	while ((cutAt = str.find_first_of(delim, Prev)) != str.npos)
	{
		if (cutAt == Prev)
		{
			// Empty string due to multiple whitespace / whitespace at the beginning of the input
			// Just skip it
			Prev = Prev + 1;
			continue;
		}
		AString current = str.substr(Prev, cutAt - Prev);
		if ((current.front() == '"') || (current.front() == '\''))
		{
			Prev += 1;
			cutAtQuote = str.find_first_of(current.front(), Prev);
			if (cutAtQuote != str.npos)
			{
				current = str.substr(Prev, cutAtQuote - Prev);
				cutAt = cutAtQuote + 1;
			}
		}

		results.push_back(std::move(current));
		Prev = cutAt + 1;
	}
{'fraction_non_alphanumeric': 0.1416327716443928, 'fraction_numerical': 0.0071179546775130735, 'mean_word_length': 3.9390243902439024, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4697150', 'n_tokens_mistral': 2792, 'n_tokens_neox': 2588, 'n_words': 663}