<commit_before>#include <errno.h>
#include <stdint.h>
#include <string.h>

#include "stringy.h"

// For unicode conversion
#include <iconv.h>

#include "debug.h"

#include "allfiles.h"
#include "moreio.h"
#include "newfatal.h"
#include "version.h"


bool sliceBusy = true;
FILE * bigDataFile = NULL;
uint32_t startOfDataIndex, startOfTextIndex,
			  startOfSubIndex, startOfObjectIndex;

bool openSubSlice (int num) {
//	FILE * dbug = fopen ("debuggy.txt", "at");

//	fprintf (dbug, "\nTrying to open sub %i\n", num);

	if (sliceBusy) {
		fatal ("Can't read from data file", "I'm already reading something");
		return false;
	}

//	fprintf (dbug, "Going to position %li\n", startOfSubIndex + (num << 2));
	fseek (bigDataFile, startOfSubIndex + (num << 2), 0);
	fseek (bigDataFile, get4bytes (bigDataFile), 0);
//	fprintf (dbug, "Told to skip forward to %li\n", ftell (bigDataFile));
//	fclose (dbug);
	return sliceBusy = true;
}

bool openObjectSlice (int num) {
//	FILE * dbug = fopen ("debuggy.txt", "at");

//	fprintf (dbug, "\nTrying to open object %i\n", num);

	if (sliceBusy) {
		fatal ("Can't read from data file", "I'm already reading something");
		return false;
	}

//	fprintf (dbug, "Going to position %li\n", startOfObjectIndex + (num << 2));
	fseek (bigDataFile, startOfObjectIndex + (num << 2), 0);
	fseek (bigDataFile, get4bytes (bigDataFile), 0);
//	fprintf (dbug, "Told to skip forward to %li\n", ftell (bigDataFile));
//	fclose (dbug);
	return sliceBusy = true;
}

unsigned int openFileFromNum (int num) {
//	FILE * dbug = fopen ("debuggy.txt", "at");

	if (sliceBusy) {
		fatal ("Can't read from data file", "I'm already reading something");
		return 0;
	}

//	fprintf (dbug, "\nTrying to open file %i\n", num);
//	fprintf (dbug, "Jumping to %li (for index) \n", startOfDataIndex + (num << 2));
	fseek (bigDataFile, startOfDataIndex + (num << 2), 0);
	fseek (bigDataFile, get4bytes (bigDataFile), 1);
//	fprintf (dbug, "Jumping to %li (for data) \n", ftell (bigDataFile));
	sliceBusy = true;
//	fclose (dbug);

	return get4bytes (bigDataFile);
}


// Converts a string from Windows CP-1252 to UTF-8. 
// This is needed for old games.
char * convertString(char * s) {
	static char *buf = NULL;
	
	if (! buf) {
		buf = new char [65536];	
		if (! checkNew (buf)) return NULL;
	}

	char **tmp1 = (char **) &s;
	char **tmp2 = (char **) &buf;
	char * sOrig = s;
	char * bufOrig = buf;
	
	iconv_t convert = iconv_open ("UTF-8", "CP1250");<commit_msg>Linux engine now converts games from ISO8859-2 to UTF-8, not from CP1250.<commit_after>#include <errno.h>
#include <stdint.h>
#include <string.h>

#include "stringy.h"

// For unicode conversion
#include <iconv.h>

#include "debug.h"

#include "allfiles.h"
#include "moreio.h"
#include "newfatal.h"
#include "version.h"


bool sliceBusy = true;
FILE * bigDataFile = NULL;
uint32_t startOfDataIndex, startOfTextIndex,
			  startOfSubIndex, startOfObjectIndex;

bool openSubSlice (int num) {
//	FILE * dbug = fopen ("debuggy.txt", "at");

//	fprintf (dbug, "\nTrying to open sub %i\n", num);

	if (sliceBusy) {
		fatal ("Can't read from data file", "I'm already reading something");
		return false;
	}

//	fprintf (dbug, "Going to position %li\n", startOfSubIndex + (num << 2));
	fseek (bigDataFile, startOfSubIndex + (num << 2), 0);
	fseek (bigDataFile, get4bytes (bigDataFile), 0);
//	fprintf (dbug, "Told to skip forward to %li\n", ftell (bigDataFile));
//	fclose (dbug);
	return sliceBusy = true;
}

bool openObjectSlice (int num) {
//	FILE * dbug = fopen ("debuggy.txt", "at");

//	fprintf (dbug, "\nTrying to open object %i\n", num);

	if (sliceBusy) {
		fatal ("Can't read from data file", "I'm already reading something");
		return false;
	}

//	fprintf (dbug, "Going to position %li\n", startOfObjectIndex + (num << 2));
	fseek (bigDataFile, startOfObjectIndex + (num << 2), 0);
	fseek (bigDataFile, get4bytes (bigDataFile), 0);
//	fprintf (dbug, "Told to skip forward to %li\n", ftell (bigDataFile));
//	fclose (dbug);
	return sliceBusy = true;
}

unsigned int openFileFromNum (int num) {
//	FILE * dbug = fopen ("debuggy.txt", "at");

	if (sliceBusy) {
		fatal ("Can't read from data file", "I'm already reading something");
		return 0;
	}

//	fprintf (dbug, "\nTrying to open file %i\n", num);
//	fprintf (dbug, "Jumping to %li (for index) \n", startOfDataIndex + (num << 2));
	fseek (bigDataFile, startOfDataIndex + (num << 2), 0);
	fseek (bigDataFile, get4bytes (bigDataFile), 1);
//	fprintf (dbug, "Jumping to %li (for data) \n", ftell (bigDataFile));
	sliceBusy = true;
//	fclose (dbug);

	return get4bytes (bigDataFile);
}


// Converts a string from Windows CP-1252 to UTF-8. 
// This is needed for old games.
char * convertString(char * s) {
	static char *buf = NULL;
	
	if (! buf) {
		buf = new char [65536];	
		if (! checkNew (buf)) return NULL;
	}

	char **tmp1 = (char **) &s;
	char **tmp2 = (char **) &buf;
	char * sOrig = s;
	char * bufOrig = buf;
#if defined __unix__ && !(defined __APPLE__)
	iconv_t convert = iconv_open ("UTF-8", "ISO8859-2");
#else
	iconv_t convert = iconv_open ("UTF-8", "CP1250");
#endif
{'fraction_non_alphanumeric': 0.16817111459968603, 'fraction_numerical': 0.01726844583987441, 'mean_word_length': 3.9775390625, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6479714', 'n_tokens_mistral': 2037, 'n_tokens_neox': 1928, 'n_words': 598}