<commit_before>#include "cbase.h"
#include "ios_fileupdater.h"
#include "curl/curl.h"
#include "Filesystem.h"
#include "utlbuffer.h"
#include "checksum_md5.h"
#include "threadtools.h"
#include "ios_teamkit_parse.h"

ConVar cl_clientfiles_download_url("cl_clientfiles_download_url", "http://simrai.iosoccer.com/downloads/clientfiles/iosoccer");
ConVar sv_serverfiles_download_url("sv_serverfiles_download_url", "http://simrai.iosoccer.com/downloads/serverfiles/iosoccer");

struct FileInfo
{
	char path[256];
	char md5[33];
};

struct TeamKitCurl
{
	FileHandle_t fh;
	MD5Context_t md5Ctx;
};

static size_t rcvFileListData(void *ptr, size_t size, size_t nmemb, CUtlBuffer &buffer)
{
	buffer.Put(ptr, nmemb);

	return size * nmemb;
}

static size_t rcvKitData(void *ptr, size_t size, size_t nmemb, TeamKitCurl *vars)
{
	filesystem->Write(ptr, nmemb, vars->fh);
	MD5Update(&vars->md5Ctx, (unsigned char *)ptr, nmemb);

	return size * nmemb;
}

void GetFileList(char *fileListString, CUtlVector<FileInfo> &fileList)
{
	char *file = strtok(fileListString, "\n");

	while (file != NULL)
	{
		FileInfo fileInfo;
		Q_strncpy(fileInfo.path, file, min(strcspn(file, ":") + 1, sizeof(fileInfo.path)));
		Q_strncpy(fileInfo.md5, strstr(file, ":") + 1, sizeof(fileInfo.md5));
//#ifdef GAME_DLL
//		if (strstr(fileInfo.path, ".txt"))
//			fileList.AddToTail(fileInfo);
//#else
		fileList.AddToTail(fileInfo);
//#endif

		file = strtok(NULL, "\n");
	}
}

unsigned PerformUpdate(void *params)
{
	const char *downloadUrl;

#ifdef CLIENT_DLL
	downloadUrl = cl_clientfiles_download_url.GetString();
#else
	downloadUrl = sv_serverfiles_download_url.GetString();
#endif

	IOSUpdateInfo *pUpdateInfo = (IOSUpdateInfo *)params;

	const char *fileListPath = "filelist.txt";

	CUtlVector<FileInfo> localFileList;

	if (filesystem->FileExists(fileListPath, "MOD"))
	{
		FileHandle_t fh = filesystem->Open(fileListPath, "r", "MOD");
		int fileSize = filesystem->Size(fh);
		char *localFileListString = new char[fileSize + 1];
 
		filesystem->Read((void *)localFileListString, fileSize, fh);
		localFileListString[fileSize] = 0; // null terminator
 
		filesystem->Close(fh);
		GetFileList(localFileListString, localFileList);
		delete[] localFileListString;
	}

	CUtlBuffer buffer;
	CURL *curl;
	curl = curl_easy_init();
	char url[512];
	Q_snprintf(url, sizeof(url), "%s/filelist.txt.gz", downloadUrl);
	curl_easy_setopt(curl, CURLOPT_URL, url);
	curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, "gzip");
	curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, rcvFileListData);
	curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buffer);
	CURLcode result = curl_easy_perform(curl);
	long code;
	curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code);
	curl_easy_cleanup(curl);

	if (result != CURLE_OK || code != 200)
	{
		pUpdateInfo->connectionError = true;
		return CFileUpdater::UpdateFinished(pUpdateInfo);
	}

	char *serverFileListString = new char[buffer.Size() + 1];
	buffer.GetString(serverFileListString);
	serverFileListString[buffer.Size()] = 0;

	CUtlVector<FileInfo> serverFileList;

	GetFileList(serverFileListString, serverFileList);

	delete[] serverFileListString;

	for (int i = 0; i < localFileList.Count(); i++)
	{
		for (int j = 0; j < serverFileList.Count(); j++)
		{
			if (!Q_strcmp(serverFileList[j].path, localFileList[i].path))
			{
				if (!Q_strcmp(serverFileList[j].md5, localFileList[i].md5))
					serverFileList.Remove(j);
				else
				{
					localFileList.Remove(i);
					i -= 1;
				}

				break;
			}
		}
	}

	pUpdateInfo->filesToUpdateCount = serverFileList.Count();

	if (pUpdateInfo->checkOnly || pUpdateInfo->filesToUpdateCount == 0)
	{
		return CFileUpdater::UpdateFinished(pUpdateInfo);
	}

	FileHandle_t fh = filesystem->Open(fileListPath, "w", "MOD");

	for (int i = 0; i < localFileList.Count(); i++)
	{
		char str[256];
		Q_snprintf(str, sizeof(str), "%s%s:%s", i == 0 ? "" : "\n", localFileList[i].path, localFileList[i].md5);
		filesystem->Write(str, strlen(str), fh);
	}

	for (int i = 0; i < serverFileList.Count(); i++)
	{
		char filePath[256];
		Q_snprintf(filePath, sizeof(filePath), "%s", serverFileList[i].path);

		char folderPath[256];
		Q_strncpy(folderPath, filePath, sizeof(folderPath));
<commit_msg>Read and write the update file list in binary mode to prevent Windows from appending a carriage return character to each line<commit_after>#include "cbase.h"
#include "ios_fileupdater.h"
#include "curl/curl.h"
#include "Filesystem.h"
#include "utlbuffer.h"
#include "checksum_md5.h"
#include "threadtools.h"
#include "ios_teamkit_parse.h"

ConVar cl_clientfiles_download_url("cl_clientfiles_download_url", "http://simrai.iosoccer.com/downloads/clientfiles/iosoccer");
ConVar sv_serverfiles_download_url("sv_serverfiles_download_url", "http://simrai.iosoccer.com/downloads/serverfiles/iosoccer");

struct FileInfo
{
	char path[256];
	char md5[33];
};

struct TeamKitCurl
{
	FileHandle_t fh;
	MD5Context_t md5Ctx;
};

static size_t rcvFileListData(void *ptr, size_t size, size_t nmemb, CUtlBuffer &buffer)
{
	buffer.Put(ptr, nmemb);

	return size * nmemb;
}

static size_t rcvKitData(void *ptr, size_t size, size_t nmemb, TeamKitCurl *vars)
{
	filesystem->Write(ptr, nmemb, vars->fh);
	MD5Update(&vars->md5Ctx, (unsigned char *)ptr, nmemb);

	return size * nmemb;
}

void GetFileList(char *fileListString, CUtlVector<FileInfo> &fileList)
{
	char *file = strtok(fileListString, "\n");

	while (file != NULL)
	{
		FileInfo fileInfo;
		Q_strncpy(fileInfo.path, file, min(strcspn(file, ":") + 1, sizeof(fileInfo.path)));
		Q_strncpy(fileInfo.md5, strstr(file, ":") + 1, sizeof(fileInfo.md5));
//#ifdef GAME_DLL
//		if (strstr(fileInfo.path, ".txt"))
//			fileList.AddToTail(fileInfo);
//#else
		fileList.AddToTail(fileInfo);
//#endif

		file = strtok(NULL, "\n");
	}
}

unsigned PerformUpdate(void *params)
{
	const char *downloadUrl;

#ifdef CLIENT_DLL
	downloadUrl = cl_clientfiles_download_url.GetString();
#else
	downloadUrl = sv_serverfiles_download_url.GetString();
#endif

	IOSUpdateInfo *pUpdateInfo = (IOSUpdateInfo *)params;

	const char *fileListPath = "filelist.txt";

	CUtlVector<FileInfo> localFileList;

	if (filesystem->FileExists(fileListPath, "MOD"))
	{
		FileHandle_t fh = filesystem->Open(fileListPath, "rb", "MOD");
		int fileSize = filesystem->Size(fh);
		char *localFileListString = new char[fileSize + 1];
 
		filesystem->Read((void *)localFileListString, fileSize, fh);
		localFileListString[fileSize] = 0; // null terminator
 
		filesystem->Close(fh);
		GetFileList(localFileListString, localFileList);
		delete[] localFileListString;
	}

	CUtlBuffer buffer;
	CURL *curl;
	curl = curl_easy_init();
	char url[512];
	Q_snprintf(url, sizeof(url), "%s/filelist.txt.gz", downloadUrl);
	curl_easy_setopt(curl, CURLOPT_URL, url);
	curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, "gzip");
	curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, rcvFileListData);
	curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buffer);
	CURLcode result = curl_easy_perform(curl);
	long code;
	curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code);
	curl_easy_cleanup(curl);

	if (result != CURLE_OK || code != 200)
	{
		pUpdateInfo->connectionError = true;
		return CFileUpdater::UpdateFinished(pUpdateInfo);
	}

	char *serverFileListString = new char[buffer.Size() + 1];
	buffer.GetString(serverFileListString);
	serverFileListString[buffer.Size()] = 0;

	CUtlVector<FileInfo> serverFileList;

	GetFileList(serverFileListString, serverFileList);

	delete[] serverFileListString;

	for (int i = 0; i < localFileList.Count(); i++)
	{
		for (int j = 0; j < serverFileList.Count(); j++)
		{
			if (!Q_strcmp(serverFileList[j].path, localFileList[i].path))
			{
				if (!Q_strcmp(serverFileList[j].md5, localFileList[i].md5))
					serverFileList.Remove(j);
				else
				{
					localFileList.Remove(i);
					i -= 1;
				}

				break;
			}
		}
	}

	pUpdateInfo->filesToUpdateCount = serverFileList.Count();

	if (pUpdateInfo->checkOnly || pUpdateInfo->filesToUpdateCount == 0)
	{
		return CFileUpdater::UpdateFinished(pUpdateInfo);
	}

	FileHandle_t fh = filesystem->Open(fileListPath, "wb", "MOD");

	for (int i = 0; i < localFileList.Count(); i++)
	{
		char str[256];
		Q_snprintf(str, sizeof(str), "%s%s:%s", i == 0 ? "" : "\n", localFileList[i].path, localFileList[i].md5);
		filesystem->Write(str, strlen(str), fh);
	}

	for (int i = 0; i < serverFileList.Count(); i++)
	{
		char filePath[256];
		Q_snprintf(filePath, sizeof(filePath), "%s", serverFileList[i].path);

		char folderPath[256];
		Q_strncpy(folderPath, filePath, sizeof(folderPath));

{'fraction_non_alphanumeric': 0.14767142522817692, 'fraction_numerical': 0.010297215071378422, 'mean_word_length': 5.534403669724771, 'pattern_counts': {'":': 4, '<': 17, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4314275', 'n_tokens_mistral': 3499, 'n_tokens_neox': 3204, 'n_words': 603}