<commit_before>/* Copyright (c) 2013-2015 <NAME>
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "nointro.h"

#include "util/crc32.h"
#include "util/table.h"
#include "util/vector.h"
#include "util/vfs.h"

#define KEY_STACK_SIZE 8

struct NoIntroDB {
	struct Table categories;
	struct Table gameCrc;
};

struct NoIntroItem {
	union {
		struct Table hash;
		char* string;
	};
	enum NoIntroItemType {
		NI_HASH,
		NI_STRING
	} type;
};

DECLARE_VECTOR(NoIntroCategory, struct NoIntroItem*);
DEFINE_VECTOR(NoIntroCategory, struct NoIntroItem*);

static void _indexU32x(struct NoIntroDB* db, struct Table* table, const char* categoryKey, const char* key) {
	struct NoIntroCategory* category = HashTableLookup(&db->categories, categoryKey);
	if (!category) {
		return;
	}
	TableInit(table, 256, 0);
	char* tmpKey = strdup(key);
	const char* keyStack[KEY_STACK_SIZE] = { tmpKey };
	size_t i;
	for (i = 1; i < KEY_STACK_SIZE; ++i) {
		char* next = strchr(keyStack[i - 1], '.');
		if (!next) {
			break;
		}
		next[0] = '\0';
		keyStack[i] = next + 1;
	}
	for (i = 0; i < NoIntroCategorySize(category); ++i) {
		struct NoIntroItem* item = *NoIntroCategoryGetPointer(category, i);
		if (!item) {
			continue;
		}
		struct NoIntroItem* keyloc = item;
		size_t s;
		for (s = 0; s < KEY_STACK_SIZE && keyStack[s]; ++s) {
			if (keyloc->type != NI_HASH) {
				keyloc = 0;
				break;
			}
			keyloc = HashTableLookup(&keyloc->hash, keyStack[s]);
			if (!keyloc) {
				break;
			}
		}
		if (!keyloc || keyloc->type != NI_STRING) {
			continue;
		}
		char* end;
		uint32_t key = strtoul(keyloc->string, &end, 16);
		if (!end || *end) {
			continue;
		}
		TableInsert(table, key, item);
	}
}

static void _itemDeinit(void* value) {
	struct NoIntroItem* item = value;
	switch (item->type) {
	case NI_STRING:
		free(item->string);
		break;
	case NI_HASH:
		HashTableDeinit(&item->hash);
		break;
	}
	free(item);
}

static void _dbDeinit(void* value) {
	struct NoIntroCategory* category = value;
	size_t i;
	for (i = 0; i < NoIntroCategorySize(category); ++i) {
		struct NoIntroItem* item = *NoIntroCategoryGetPointer(category, i);
		switch (item->type) {
		case NI_STRING:
			free(item->string);
			break;
		case NI_HASH:
			HashTableDeinit(&item->hash);
			break;
		}
		free(item);
	}
	NoIntroCategoryDeinit(category);
}

static bool _itemToGame(const struct NoIntroItem* item, struct NoIntroGame* game) {
	if (item->type != NI_HASH) {
		return false;
	}
	struct NoIntroItem* subitem;
	struct NoIntroItem* rom;

	memset(game, 0, sizeof(*game));
	subitem = HashTableLookup(&item->hash, "name");
	if (subitem && subitem->type == NI_STRING) {
		game->name = subitem->string;
	}
	subitem = HashTableLookup(&item->hash, "description");
	if (subitem && subitem->type == NI_STRING) {
		game->description = subitem->string;
	}

	rom = HashTableLookup(&item->hash, "rom");
	if (!rom || rom->type != NI_HASH) {
		return false;
	}
	subitem = HashTableLookup(&rom->hash, "name");
	if (subitem && subitem->type == NI_STRING) {
		game->romName = subitem->string;
	}
	subitem = HashTableLookup(&rom->hash, "size");
	if (subitem && subitem->type == NI_STRING) {
		char* end;
		game->size = strtoul(subitem->string, &end, 0);
		if (!end || *end) {
			game->size = 0;
		}
	}
	// TODO: md5, sha1
	subitem = HashTableLookup(&rom->hash, "flags");
	if (subitem && subitem->type == NI_STRING && strcmp(subitem->string, "verified")) {
		game->verified = true;
	}

	return true;
}

struct NoIntroDB* NoIntroDBLoad(struct VFile* vf) {
	struct NoIntroDB* db = malloc(sizeof(*db));
	HashTableInit(&db->categories, 0, _dbDeinit);
	char line[512];
	struct {
		char* key;
		struct NoIntroItem* item;
	} keyStack[KEY_STACK_SIZE];
	struct Table* parent = 0;

	size_t stackDepth = 0;
	while (true) {
		ssize_t bytesRead = vf->readline(vf, line, sizeof(line));
		if (!bytesRead) {
			break;
		}
		ssize_t i;
		const char* token;
		for (i = 0; i < bytesRead; ++i) {
			while (isspace((int) line[i]) && i < bytesRead) {
				++i;
			}
			if (i >= bytesRead) {
				break;<commit_msg>Util: Initialize No-Intro DB parsing stack<commit_after>/* Copyright (c) 2013-2015 <NAME>
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "nointro.h"

#include "util/crc32.h"
#include "util/table.h"
#include "util/vector.h"
#include "util/vfs.h"

#define KEY_STACK_SIZE 8

struct NoIntroDB {
	struct Table categories;
	struct Table gameCrc;
};

struct NoIntroItem {
	union {
		struct Table hash;
		char* string;
	};
	enum NoIntroItemType {
		NI_HASH,
		NI_STRING
	} type;
};

DECLARE_VECTOR(NoIntroCategory, struct NoIntroItem*);
DEFINE_VECTOR(NoIntroCategory, struct NoIntroItem*);

static void _indexU32x(struct NoIntroDB* db, struct Table* table, const char* categoryKey, const char* key) {
	struct NoIntroCategory* category = HashTableLookup(&db->categories, categoryKey);
	if (!category) {
		return;
	}
	TableInit(table, 256, 0);
	char* tmpKey = strdup(key);
	const char* keyStack[KEY_STACK_SIZE] = { tmpKey };
	size_t i;
	for (i = 1; i < KEY_STACK_SIZE; ++i) {
		char* next = strchr(keyStack[i - 1], '.');
		if (!next) {
			break;
		}
		next[0] = '\0';
		keyStack[i] = next + 1;
	}
	for (i = 0; i < NoIntroCategorySize(category); ++i) {
		struct NoIntroItem* item = *NoIntroCategoryGetPointer(category, i);
		if (!item) {
			continue;
		}
		struct NoIntroItem* keyloc = item;
		size_t s;
		for (s = 0; s < KEY_STACK_SIZE && keyStack[s]; ++s) {
			if (keyloc->type != NI_HASH) {
				keyloc = 0;
				break;
			}
			keyloc = HashTableLookup(&keyloc->hash, keyStack[s]);
			if (!keyloc) {
				break;
			}
		}
		if (!keyloc || keyloc->type != NI_STRING) {
			continue;
		}
		char* end;
		uint32_t key = strtoul(keyloc->string, &end, 16);
		if (!end || *end) {
			continue;
		}
		TableInsert(table, key, item);
	}
}

static void _itemDeinit(void* value) {
	struct NoIntroItem* item = value;
	switch (item->type) {
	case NI_STRING:
		free(item->string);
		break;
	case NI_HASH:
		HashTableDeinit(&item->hash);
		break;
	}
	free(item);
}

static void _dbDeinit(void* value) {
	struct NoIntroCategory* category = value;
	size_t i;
	for (i = 0; i < NoIntroCategorySize(category); ++i) {
		struct NoIntroItem* item = *NoIntroCategoryGetPointer(category, i);
		switch (item->type) {
		case NI_STRING:
			free(item->string);
			break;
		case NI_HASH:
			HashTableDeinit(&item->hash);
			break;
		}
		free(item);
	}
	NoIntroCategoryDeinit(category);
}

static bool _itemToGame(const struct NoIntroItem* item, struct NoIntroGame* game) {
	if (item->type != NI_HASH) {
		return false;
	}
	struct NoIntroItem* subitem;
	struct NoIntroItem* rom;

	memset(game, 0, sizeof(*game));
	subitem = HashTableLookup(&item->hash, "name");
	if (subitem && subitem->type == NI_STRING) {
		game->name = subitem->string;
	}
	subitem = HashTableLookup(&item->hash, "description");
	if (subitem && subitem->type == NI_STRING) {
		game->description = subitem->string;
	}

	rom = HashTableLookup(&item->hash, "rom");
	if (!rom || rom->type != NI_HASH) {
		return false;
	}
	subitem = HashTableLookup(&rom->hash, "name");
	if (subitem && subitem->type == NI_STRING) {
		game->romName = subitem->string;
	}
	subitem = HashTableLookup(&rom->hash, "size");
	if (subitem && subitem->type == NI_STRING) {
		char* end;
		game->size = strtoul(subitem->string, &end, 0);
		if (!end || *end) {
			game->size = 0;
		}
	}
	// TODO: md5, sha1
	subitem = HashTableLookup(&rom->hash, "flags");
	if (subitem && subitem->type == NI_STRING && strcmp(subitem->string, "verified")) {
		game->verified = true;
	}

	return true;
}

struct NoIntroDB* NoIntroDBLoad(struct VFile* vf) {
	struct NoIntroDB* db = malloc(sizeof(*db));
	HashTableInit(&db->categories, 0, _dbDeinit);
	char line[512];
	struct {
		char* key;
		struct NoIntroItem* item;
	} keyStack[KEY_STACK_SIZE] = {};
	struct Table* parent = 0;

	size_t stackDepth = 0;
	while (true) {
		ssize_t bytesRead = vf->readline(vf, line, sizeof(line));
		if (!bytesRead) {
			break;
		}
		ssize_t i;
		const char* token;
		for (i = 0; i < bytesRead; ++i) {
			while (isspace((int) line[i]) && i < bytesRead) {
				++i;
			}
			if (i >= bytesRead) {
				break;
{'fraction_non_alphanumeric': 0.15800094966761632, 'fraction_numerical': 0.010921177587844255, 'mean_word_length': 3.89540964555491, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 81, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2803987', 'n_tokens_mistral': 3549, 'n_tokens_neox': 3254, 'n_words': 790}