<commit_before>/*
	OpenLieroX

	string utilities
	
	code under LGPL
	created 01-05-2007
	by <NAME> and <NAME>
*/

#ifndef __STRINGUTILS_H__
#define __STRINGUTILS_H__

#include <SDL.h> // for Uint32
#include <cstdio> // for FILE
#include <string>
#include <sstream>
#include <vector>
#include <cassert>
#include <list>
#include <limits.h>
#include "types.h"
#include "Color.h" // for StrToCol
#include "Iterator.h"

//
// C-string handling routines
//
// HINT: these are obsolete, use std::string where possible!!!

// Secure c-string handling macros
// WARNING: don't use expressions like buf[i++] with the macros, because the "i" variable will be incremented twice in some macros!
#define		fix_markend(chrarray) \
				chrarray[sizeof(chrarray)-1] = '\0';
#define		fix_strnlen(chrarray) \
				strnlen(chrarray,sizeof(chrarray))
#define		fix_strncpy(chrarray, src) \
			{	strncpy(chrarray, src, sizeof(chrarray)); \
			 	chrarray[sizeof(chrarray)-1] = '\0'; }
#define		fix_strncat(chrarray, src) \
			{	size_t destlen = strnlen(chrarray, sizeof(chrarray)); \
				strncpy(&chrarray[destlen], src, sizeof(chrarray)-destlen); \
				chrarray[sizeof(chrarray)-1] = '\0'; }
#define		dyn_markend(dest, len) \
				dest[len-1] = '\0';
#define		dyn_strncpy(dest, src, len) \
			{	strncpy(dest, src, len); \
				dest[len-1] = '\0'; }
#define		dyn_strncat(dest, src, len) \
			{	size_t destlen = strnlen(dest, len); \
				strncpy(&dest[destlen], src, len-destlen); \
				dest[len-1] = '\0'; }


// Strnlen definition for compilers that don't have it
#ifndef __USE_GNU
	inline size_t strnlen(const char *str, size_t maxlen) {
		register size_t i;
		for(i = 0; (i < maxlen) && str[i]; ++i) {}
		return i;
	}
#endif

// Misc cross-compiler compatibility problem solutions
#ifdef WIN32
#if (defined(_MSC_VER) && (_MSC_VER <= 1200))
	inline int strncasecmp(const char *str1, const char *str2, size_t l) {
		return _strnicmp(str1, str2, l);
	}
#endif
#	define vsnprintf _vsnprintf
#	define snprintf	 _snprintf
#	define stricmp _stricmp
#	define fcloseall _fcloseall
#	define strcasecmp	stricmp
#else
inline char* strlwr(char* string) {
	char* ret = string;
	if(string) while( 0 != ( *string++ = (char)tolower( *string ) ) ) ;
	return ret;
}
#endif


/////////////
// Case-insensitive comparison of two chars, behaves like stringcasecmp
int chrcasecmp(const char c1, const char c2);

/////////////
// C-string itoa for non-windows compilers (on Windows it's defined in windows.h)
#ifndef WIN32
// TODOL remove this
inline char* itoa(int val, char* buf, int base) {
	int i = 29; // TODO: bad style
	buf[i+1] = '\0';

    do {
        buf = "0123456789abcdefghijklmnopqrstuvwxyz"[val % base] + buf;
        --i, val /= base;
    } while(val && i);

    return &buf[i+1];
}

// Cross-compiler compatibility
#	define		stricmp		strcasecmp
#endif


//
// C++ string (std::string) routines
//
// HINT: use these where possible

void			TrimSpaces(std::string& szLine);
bool			replace(const std::string& text, const std::string& what, const std::string& with, std::string& result);
bool			replace(std::string& text, const std::string& what, const std::string& with);
std::string		replacemax(const std::string& text, const std::string& what, const std::string& with, std::string& result, int max);
std::string		replacemax(const std::string& text, const std::string& what, const std::string& with, int max);
std::string		strip(const std::string& text, int width);
bool			stripdot(std::string& text, int width);
void			ucfirst(std::string& text);
std::string		ReadUntil(const std::string& text, char until_character = '\n'); // will return whole text if not found
std::string		ReadUntil(const std::string& text, std::string::const_iterator& start, char until_character, const std::string& alternative = "");
std::string		ReadUntil(FILE* fp, char until_character = '\n');
Color			StrToCol(const std::string& str);
Color			StrToCol(const std::string& str, bool& fail);
std::vector<std::string> explode(const std::string& str, const std::string& delim);
void			freadstr(std::string& result, size_t maxlen, FILE *fp);
size_t			fwrite(const std::string& txt, size_t len, FILE* fp);
size_t			findLastPathSep(const std::string& path);
void			stringlwr(std::string& txt);
std::string		stringtolower(const std::string& txt);
bool			strincludes(const std::string& str, const std::string& what);
short			stringcasecmp(const std::string& s1, const std::string& s2);
bool			stringcaseequal(const std::string& s1, const std::string& s2);
bool			subStrEqual(const std::string& s1, const std::string s2, size_t p);
bool			subStrCaseEqual(const std::string& s1, const std::string s2, size_t p);
inline bool		strStartsWith(const std::string& str, const std::string& start) { if(start.size() > str.size()) return false; return str.substr(0,start.size()) == start; }
size_t			maxStartingEqualStr(const std::list<std::string>& strs);
std::vector<std::string> splitstring(const std::string& str, size_t maxlen, size_t maxwidth, class CFont& font);
std::string		GetFileExtension(const std::string& filename);
std::string		GetBaseFilename(const std::string& filename);
size_t			stringcasefind(const std::string& text, const std::string& search_for);
size_t			stringcaserfind(const std::string& text, const std::string& search_for);
std::string		StripHtmlTags( const std::string & src );	// Also removes all "\r" and spaces at line beginning
std::string		GetNextWord(std::string::const_iterator it, const std::string& str);
bool 			Compress( const std::string & in, std::string * out, bool noCompression = false );	// Compress given string using zlib, noCompression will just add zlib header and checksum
bool 			Decompress( const std::string & in, std::string * out );	// Decompress, returns false if checksum fails
size_t			StringChecksum( const std::string & data );
bool			FileChecksum( const std::string & path, size_t * _checksum, size_t * _filesize );
std::string		Base64Encode(const std::string &data);
std::string		UrlEncode(const std::string &data); // Substitute space with + and all non-alphanum symbols with %XX
std::string		AutoDetectLinks(const std::string& text);
std::string		HtmlEntityUnpairedBrackets(const std::string &txt);
size_t			GetPosByTextWidth(const std::string& text, int width, CFont *fnt);
std::string		ColToHex(Uint32 col);
std::string		EscapeHtmlTags( const std::string & src );	// Escape all "<" and ">" and "&"

bool			strSeemsLikeChatCommand(const std::string& str);

inline size_t subStrCount(const std::string& str, const std::string& substr) {
	size_t c = 0;
	size_t p = 0;
	while((p = str.find(substr, p)) != std::string::npos)
		c++;
	return c;
}


typedef void (*PrintOutFct) (const std::string&);
inline void NullOut(const std::string&) {}

// returns true if last char was a newline
bool PrettyPrint(const std::string& prefix, const std::string& buf, PrintOutFct printOutFct, bool firstLineWithPrefix = true);


Iterator<char>::Ref HexDump(Iterator<char>::Ref start, PrintOutFct printOutFct, size_t mark = (size_t)-1, size_t count = (size_t)-1);

<commit_msg>small fix<commit_after>/*
	OpenLieroX

	string utilities
	
	code under LGPL
	created 01-05-2007
	by <NAME> and <NAME>
*/

#ifndef __STRINGUTILS_H__
#define __STRINGUTILS_H__

#include <SDL.h> // for Uint32
#include <cstdio> // for FILE
#include <string>
#include <sstream>
#include <vector>
#include <cassert>
#include <list>
#include <limits.h>
#include "types.h"
#include "Color.h" // for StrToCol
#include "Iterator.h"

//
// C-string handling routines
//
// HINT: these are obsolete, use std::string where possible!!!

// Secure c-string handling macros
// WARNING: don't use expressions like buf[i++] with the macros, because the "i" variable will be incremented twice in some macros!
#define		fix_markend(chrarray) \
				chrarray[sizeof(chrarray)-1] = '\0';
#define		fix_strnlen(chrarray) \
				strnlen(chrarray,sizeof(chrarray))
#define		fix_strncpy(chrarray, src) \
			{	strncpy(chrarray, src, sizeof(chrarray)); \
			 	chrarray[sizeof(chrarray)-1] = '\0'; }
#define		fix_strncat(chrarray, src) \
			{	size_t destlen = strnlen(chrarray, sizeof(chrarray)); \
				strncpy(&chrarray[destlen], src, sizeof(chrarray)-destlen); \
				chrarray[sizeof(chrarray)-1] = '\0'; }
#define		dyn_markend(dest, len) \
				dest[len-1] = '\0';
#define		dyn_strncpy(dest, src, len) \
			{	strncpy(dest, src, len); \
				dest[len-1] = '\0'; }
#define		dyn_strncat(dest, src, len) \
			{	size_t destlen = strnlen(dest, len); \
				strncpy(&dest[destlen], src, len-destlen); \
				dest[len-1] = '\0'; }


// Strnlen definition for compilers that don't have it
#ifndef __USE_GNU
	inline size_t strnlen(const char *str, size_t maxlen) {
		register size_t i;
		for(i = 0; (i < maxlen) && str[i]; ++i) {}
		return i;
	}
#endif

// Misc cross-compiler compatibility problem solutions
#ifdef WIN32
#if (defined(_MSC_VER) && (_MSC_VER <= 1200))
	inline int strncasecmp(const char *str1, const char *str2, size_t l) {
		return _strnicmp(str1, str2, l);
	}
#endif
#	define vsnprintf _vsnprintf
#	define snprintf	 _snprintf
#	define stricmp _stricmp
#	define fcloseall _fcloseall
#	define strcasecmp	stricmp
#else
inline char* strlwr(char* string) {
	char* ret = string;
	if(string) while( 0 != ( *string++ = (char)tolower( *string ) ) ) ;
	return ret;
}
#endif


/////////////
// Case-insensitive comparison of two chars, behaves like stringcasecmp
int chrcasecmp(const char c1, const char c2);

/////////////
// C-string itoa for non-windows compilers (on Windows it's defined in windows.h)
#ifndef WIN32
// TODOL remove this
inline char* itoa(int val, char* buf, int base) {
	int i = 29; // TODO: bad style
	buf[i+1] = '\0';

    do {
        buf = "0123456789abcdefghijklmnopqrstuvwxyz"[val % base] + buf;
        --i, val /= base;
    } while(val && i);

    return &buf[i+1];
}

// Cross-compiler compatibility
#	define		stricmp		strcasecmp
#endif


//
// C++ string (std::string) routines
//
// HINT: use these where possible

void			TrimSpaces(std::string& szLine);
bool			replace(const std::string& text, const std::string& what, const std::string& with, std::string& result);
bool			replace(std::string& text, const std::string& what, const std::string& with);
std::string		replacemax(const std::string& text, const std::string& what, const std::string& with, std::string& result, int max);
std::string		replacemax(const std::string& text, const std::string& what, const std::string& with, int max);
std::string		strip(const std::string& text, int width);
bool			stripdot(std::string& text, int width);
void			ucfirst(std::string& text);
std::string		ReadUntil(const std::string& text, char until_character = '\n'); // will return whole text if not found
std::string		ReadUntil(const std::string& text, std::string::const_iterator& start, char until_character, const std::string& alternative = "");
std::string		ReadUntil(FILE* fp, char until_character = '\n');
Color			StrToCol(const std::string& str);
Color			StrToCol(const std::string& str, bool& fail);
std::vector<std::string> explode(const std::string& str, const std::string& delim);
void			freadstr(std::string& result, size_t maxlen, FILE *fp);
size_t			fwrite(const std::string& txt, size_t len, FILE* fp);
size_t			findLastPathSep(const std::string& path);
void			stringlwr(std::string& txt);
std::string		stringtolower(const std::string& txt);
bool			strincludes(const std::string& str, const std::string& what);
short			stringcasecmp(const std::string& s1, const std::string& s2);
bool			stringcaseequal(const std::string& s1, const std::string& s2);
bool			subStrEqual(const std::string& s1, const std::string s2, size_t p);
bool			subStrCaseEqual(const std::string& s1, const std::string s2, size_t p);
inline bool		strStartsWith(const std::string& str, const std::string& start) { if(start.size() > str.size()) return false; return str.substr(0,start.size()) == start; }
size_t			maxStartingEqualStr(const std::list<std::string>& strs);
std::vector<std::string> splitstring(const std::string& str, size_t maxlen, size_t maxwidth, class CFont& font);
std::string		GetFileExtension(const std::string& filename);
std::string		GetBaseFilename(const std::string& filename);
size_t			stringcasefind(const std::string& text, const std::string& search_for);
size_t			stringcaserfind(const std::string& text, const std::string& search_for);
std::string		StripHtmlTags( const std::string & src );	// Also removes all "\r" and spaces at line beginning
std::string		GetNextWord(std::string::const_iterator it, const std::string& str);
bool 			Compress( const std::string & in, std::string * out, bool noCompression = false );	// Compress given string using zlib, noCompression will just add zlib header and checksum
bool 			Decompress( const std::string & in, std::string * out );	// Decompress, returns false if checksum fails
size_t			StringChecksum( const std::string & data );
bool			FileChecksum( const std::string & path, size_t * _checksum, size_t * _filesize );
std::string		Base64Encode(const std::string &data);
std::string		UrlEncode(const std::string &data); // Substitute space with + and all non-alphanum symbols with %XX
std::string		AutoDetectLinks(const std::string& text);
std::string		HtmlEntityUnpairedBrackets(const std::string &txt);
size_t			GetPosByTextWidth(const std::string& text, int width, CFont *fnt);
std::string		ColToHex(Uint32 col);
std::string		EscapeHtmlTags( const std::string & src );	// Escape all "<" and ">" and "&"

bool			strSeemsLikeChatCommand(const std::string& str);

inline size_t subStrCount(const std::string& str, const std::string& substr) {
	size_t c = 0, p = 0;
	while((p = str.find(substr, p)) != std::string::npos) { c++; p++; }
	return c;
}


typedef void (*PrintOutFct) (const std::string&);
inline void NullOut(const std::string&) {}

// returns true if last char was a newline
bool PrettyPrint(const std::string& prefix, const std::string& buf, PrintOutFct printOutFct, bool firstLineWithPrefix = true);


Iterator<char>::Ref HexDump(Iterator<char>::Ref start, PrintOutFct printOutFct, size_t mark = (size_t)-1, size_t count = (size_t)-1);


{'fraction_non_alphanumeric': 0.147582877830624, 'fraction_numerical': 0.00993824093135515, 'mean_word_length': 4.916841663166736, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2842929', 'n_tokens_mistral': 5116, 'n_tokens_neox': 4853, 'n_words': 1409}