<commit_before>#include <sys/types.h>
#include <sys/sbuf.h>

#include <assert.h>
#include <ctype.h>
#include <errno.h>
#include <inttypes.h>
#include <stdbool.h>
#include <stdlib.h>
#include <string.h>
#include <yaml.h>

#include "pkg.h"
#include "pkg_event.h"
#include "pkg_util.h"
#include "pkg_private.h"

#define PKG_UNKNOWN -1
#define PKG_DEPS -2
#define PKG_CONFLICTS -3
#define PKG_FILES -4
#define PKG_DIRS -5
#define PKG_FLATSIZE -6
#define PKG_SCRIPTS -7

static void parse_mapping(struct pkg *, yaml_node_pair_t *, yaml_document_t *, int);
static void parse_node(struct pkg *, yaml_node_t *, yaml_document_t *, int);

static struct manifest_key {
	const char *key;
	int type;
} manifest_key[] = {
	{ "name", PKG_NAME },
	{ "origin", PKG_ORIGIN },
	{ "version", PKG_VERSION },
	{ "arch", PKG_ARCH },
	{ "osversion", PKG_OSVERSION },
	{ "www", PKG_WWW },
	{ "comment", PKG_COMMENT},
	{ "maintainer", PKG_MAINTAINER},
	{ "prefix", PKG_PREFIX},
	{ "deps", PKG_DEPS},
	{ "conflicts", PKG_CONFLICTS},
	{ "files", PKG_FILES},
	{ "dirs", PKG_DIRS},
	{ "flatsize", PKG_FLATSIZE},
	{ "desc", PKG_DESC },
	{ "scripts", PKG_SCRIPTS},
	{ "message", PKG_MESSAGE},
};

#define manifest_key_len (int)(sizeof(manifest_key)/sizeof(manifest_key[0]))

static int manifest_type(const char *key) {
	int i = 0;
	
	for (i = 0; i < manifest_key_len; i++) {
		if (!strcasecmp(key, manifest_key[i].key))
			return (manifest_key[i].type);
	}

	return (PKG_UNKNOWN);
}

int
pkg_load_manifest_file(struct pkg *pkg, const char *fpath)
{
	char *manifest = NULL;
	off_t sz;
	int ret = EPKG_OK;

	if ((ret = file_to_buffer(fpath, &manifest, &sz)) != EPKG_OK)
		return (ret);

	ret = pkg_parse_manifest(pkg, manifest);
	free(manifest);

	return (ret);
}

static void
parse_mapping(struct pkg *pkg, yaml_node_pair_t *pair, yaml_document_t *document, int pkgtype)
{
	int type;
	yaml_node_t *key, *subkey;
	yaml_node_t *val, *subval;
	yaml_node_pair_t *subpair;
	char origin[BUFSIZ];
	char version[BUFSIZ];
	pkg_script_t script_type;

	key = yaml_document_get_node(document, pair->key);
	val = yaml_document_get_node(document, pair->value);

	switch (pkgtype) {
		case PKG_FILES:
			pkg_addfile(pkg, key->data.scalar.value, val->data.scalar.length == 65 ? val->data.scalar.value : NULL);
			break;
		case PKG_DEPS:
			subpair = val->data.mapping.pairs.start;
			while (subpair < val->data.mapping.pairs.top) {
				subkey = yaml_document_get_node(document, subpair->key);
				subval = yaml_document_get_node(document, subpair->value);
				if (!strcasecmp(subkey->data.scalar.value, "origin"))
					strlcpy(origin, subval->data.scalar.value, BUFSIZ);
				else if (!strcasecmp(subkey->data.scalar.value, "version"))
					strlcpy(version, subval->data.scalar.value, BUFSIZ);
				else
					EMIT_PKG_ERROR("Ignoring key: (%s: %s) for dependency %s",
								   subkey->data.scalar.value,
								   subval->data.scalar.value,
								   key->data.scalar.value);
				++subpair;
			}
			pkg_adddep(pkg, key->data.scalar.value, origin, version);
			break;
		case PKG_SCRIPTS:
			if (strcmp(key->data.scalar.value, "pre-install") == 0) {
				script_type = PKG_SCRIPT_PRE_INSTALL;
			} else if (strcmp(key->data.scalar.value, "install") == 0) {
				script_type = PKG_SCRIPT_INSTALL;
			} else if (strcmp(key->data.scalar.value, "post-install") == 0) {
				script_type = PKG_SCRIPT_POST_INSTALL;
			} else if (strcmp(key->data.scalar.value, "pre-upgrade") == 0) {
				script_type = PKG_SCRIPT_PRE_UPGRADE;
			} else if (strcmp(key->data.scalar.value, "upgrade") == 0) {
				script_type = PKG_SCRIPT_UPGRADE;
			} else if (strcmp(key->data.scalar.value, "post-upgrade") == 0) {
				script_type = PKG_SCRIPT_POST_UPGRADE;
			} else if (strcmp(key->data.scalar.value, "pre-deinstall") == 0) {
				script_type = PKG_SCRIPT_PRE_DEINSTALL;
			} else if (strcmp(key->data.scalar.value, "deinstall") == 0) {
				script_type = PKG_SCRIPT_DEINSTALL;
			} else if (strcmp(key->data.scalar.value, "post-deinstall") == 0) {
				script_type = PKG_SCRIPT_POST_DEINSTALL;
			}

			pkg_addscript(pkg, val->data.scalar.value, script_type);
			break;
		default:
			type = manifest_type(key->data.scalar.value);
			if (type == -1) {
				if (val->type == YAML_SCALAR_NODE)
					EMIT_PKG_ERROR("Unknown line: (%s: %s)\n",
								   key->data.scalar.value,
								   val->data.scalar.value);
				else
					EMIT_PKG_ERROR("Unknown key: (%s)\n",
								   key->data.scalar.value);
				++pair;
				break;
			}
			if (val->type == YAML_SCALAR_NODE) {
				/* just ignore empty lines */
				if (val->data.scalar.length <= 0)
					break;
				type = manifest_type(key->data.scalar.value);
				if (type == -1) {
					EMIT_PKG_ERROR("Unknown line: (%s: %s)\n",
								   key->data.scalar.value,
								   val->data.scalar.value);
				} else if (type == PKG_FLATSIZE)
					pkg_setflatsize(pkg, strtoimax(val->data.scalar.value, NULL, 10));
				else {
					if (val->data.scalar.value[val->data.scalar.length - 1] == '\n')
						val->data.scalar.value[val->data.scalar.length - 1] = '\0';

					pkg_set(pkg, type, val->data.scalar.value);
				}
			} else {
				parse_node(pkg, val, document, type);
			}
			break;
	}
}

static void
parse_node(struct pkg *pkg, yaml_node_t *node, yaml_document_t *document, int pkgtype)
{
	yaml_node_pair_t *pair;
	yaml_node_item_t *item;
	yaml_node_t *nd;

	switch (node->type) {
		case YAML_SCALAR_NODE:
			/* NOT REACHED THERE SHOULD NOT BE ALONE SCALARS */
			printf("%s\n", (char *)node->data.scalar.value);
			break;
		case YAML_SEQUENCE_NODE:
			switch (pkgtype) {
				case PKG_DIRS:
					item = node->data.sequence.items.start;<commit_msg>Deal with the case when there are more than one \n<commit_after>#include <sys/types.h>
#include <sys/sbuf.h>

#include <assert.h>
#include <ctype.h>
#include <errno.h>
#include <inttypes.h>
#include <stdbool.h>
#include <stdlib.h>
#include <string.h>
#include <yaml.h>

#include "pkg.h"
#include "pkg_event.h"
#include "pkg_util.h"
#include "pkg_private.h"

#define PKG_UNKNOWN -1
#define PKG_DEPS -2
#define PKG_CONFLICTS -3
#define PKG_FILES -4
#define PKG_DIRS -5
#define PKG_FLATSIZE -6
#define PKG_SCRIPTS -7

static void parse_mapping(struct pkg *, yaml_node_pair_t *, yaml_document_t *, int);
static void parse_node(struct pkg *, yaml_node_t *, yaml_document_t *, int);

static struct manifest_key {
	const char *key;
	int type;
} manifest_key[] = {
	{ "name", PKG_NAME },
	{ "origin", PKG_ORIGIN },
	{ "version", PKG_VERSION },
	{ "arch", PKG_ARCH },
	{ "osversion", PKG_OSVERSION },
	{ "www", PKG_WWW },
	{ "comment", PKG_COMMENT},
	{ "maintainer", PKG_MAINTAINER},
	{ "prefix", PKG_PREFIX},
	{ "deps", PKG_DEPS},
	{ "conflicts", PKG_CONFLICTS},
	{ "files", PKG_FILES},
	{ "dirs", PKG_DIRS},
	{ "flatsize", PKG_FLATSIZE},
	{ "desc", PKG_DESC },
	{ "scripts", PKG_SCRIPTS},
	{ "message", PKG_MESSAGE},
};

#define manifest_key_len (int)(sizeof(manifest_key)/sizeof(manifest_key[0]))

static int manifest_type(const char *key) {
	int i = 0;
	
	for (i = 0; i < manifest_key_len; i++) {
		if (!strcasecmp(key, manifest_key[i].key))
			return (manifest_key[i].type);
	}

	return (PKG_UNKNOWN);
}

int
pkg_load_manifest_file(struct pkg *pkg, const char *fpath)
{
	char *manifest = NULL;
	off_t sz;
	int ret = EPKG_OK;

	if ((ret = file_to_buffer(fpath, &manifest, &sz)) != EPKG_OK)
		return (ret);

	ret = pkg_parse_manifest(pkg, manifest);
	free(manifest);

	return (ret);
}

static void
parse_mapping(struct pkg *pkg, yaml_node_pair_t *pair, yaml_document_t *document, int pkgtype)
{
	int type;
	yaml_node_t *key, *subkey;
	yaml_node_t *val, *subval;
	yaml_node_pair_t *subpair;
	char origin[BUFSIZ];
	char version[BUFSIZ];
	pkg_script_t script_type;

	key = yaml_document_get_node(document, pair->key);
	val = yaml_document_get_node(document, pair->value);

	switch (pkgtype) {
		case PKG_FILES:
			pkg_addfile(pkg, key->data.scalar.value, val->data.scalar.length == 65 ? val->data.scalar.value : NULL);
			break;
		case PKG_DEPS:
			subpair = val->data.mapping.pairs.start;
			while (subpair < val->data.mapping.pairs.top) {
				subkey = yaml_document_get_node(document, subpair->key);
				subval = yaml_document_get_node(document, subpair->value);
				if (!strcasecmp(subkey->data.scalar.value, "origin"))
					strlcpy(origin, subval->data.scalar.value, BUFSIZ);
				else if (!strcasecmp(subkey->data.scalar.value, "version"))
					strlcpy(version, subval->data.scalar.value, BUFSIZ);
				else
					EMIT_PKG_ERROR("Ignoring key: (%s: %s) for dependency %s",
								   subkey->data.scalar.value,
								   subval->data.scalar.value,
								   key->data.scalar.value);
				++subpair;
			}
			pkg_adddep(pkg, key->data.scalar.value, origin, version);
			break;
		case PKG_SCRIPTS:
			if (strcmp(key->data.scalar.value, "pre-install") == 0) {
				script_type = PKG_SCRIPT_PRE_INSTALL;
			} else if (strcmp(key->data.scalar.value, "install") == 0) {
				script_type = PKG_SCRIPT_INSTALL;
			} else if (strcmp(key->data.scalar.value, "post-install") == 0) {
				script_type = PKG_SCRIPT_POST_INSTALL;
			} else if (strcmp(key->data.scalar.value, "pre-upgrade") == 0) {
				script_type = PKG_SCRIPT_PRE_UPGRADE;
			} else if (strcmp(key->data.scalar.value, "upgrade") == 0) {
				script_type = PKG_SCRIPT_UPGRADE;
			} else if (strcmp(key->data.scalar.value, "post-upgrade") == 0) {
				script_type = PKG_SCRIPT_POST_UPGRADE;
			} else if (strcmp(key->data.scalar.value, "pre-deinstall") == 0) {
				script_type = PKG_SCRIPT_PRE_DEINSTALL;
			} else if (strcmp(key->data.scalar.value, "deinstall") == 0) {
				script_type = PKG_SCRIPT_DEINSTALL;
			} else if (strcmp(key->data.scalar.value, "post-deinstall") == 0) {
				script_type = PKG_SCRIPT_POST_DEINSTALL;
			}

			pkg_addscript(pkg, val->data.scalar.value, script_type);
			break;
		default:
			type = manifest_type(key->data.scalar.value);
			if (type == -1) {
				if (val->type == YAML_SCALAR_NODE)
					EMIT_PKG_ERROR("Unknown line: (%s: %s)\n",
								   key->data.scalar.value,
								   val->data.scalar.value);
				else
					EMIT_PKG_ERROR("Unknown key: (%s)\n",
								   key->data.scalar.value);
				++pair;
				break;
			}
			if (val->type == YAML_SCALAR_NODE) {
				/* just ignore empty lines */
				if (val->data.scalar.length <= 0)
					break;
				type = manifest_type(key->data.scalar.value);
				if (type == -1) {
					EMIT_PKG_ERROR("Unknown line: (%s: %s)\n",
								   key->data.scalar.value,
								   val->data.scalar.value);
				} else if (type == PKG_FLATSIZE)
					pkg_setflatsize(pkg, strtoimax(val->data.scalar.value, NULL, 10));
				else {
					while (val->data.scalar.length > 0 &&
						   val->data.scalar.value[val->data.scalar.length - 1] == '\n') {
						val->data.scalar.value[val->data.scalar.length - 1] = '\0';
						val->data.scalar.length--;
					}

					pkg_set(pkg, type, val->data.scalar.value);
				}
			} else {
				parse_node(pkg, val, document, type);
			}
			break;
	}
}

static void
parse_node(struct pkg *pkg, yaml_node_t *node, yaml_document_t *document, int pkgtype)
{
	yaml_node_pair_t *pair;
	yaml_node_item_t *item;
	yaml_node_t *nd;

	switch (node->type) {
		case YAML_SCALAR_NODE:
			/* NOT REACHED THERE SHOULD NOT BE ALONE SCALARS */
			printf("%s\n", (char *)node->data.scalar.value);
			break;
		case YAML_SEQUENCE_NODE:
			switch (pkgtype) {
				case PKG_DIRS:
					item = node->data.sequence.items.start;
{'fraction_non_alphanumeric': 0.15864197530864196, 'fraction_numerical': 0.0052028218694885366, 'mean_word_length': 3.78321383382539, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 118, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '76397', 'n_tokens_mistral': 5095, 'n_tokens_neox': 4389, 'n_words': 860}