<commit_before>/*
 * Copyright (c) 2011-2012 <NAME> <<EMAIL>>
 * Copyright (c) 2011-2012 <NAME> <<EMAIL>>
 * All rights reserved.
 * 
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer
 *    in this position and unchanged.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include <sys/types.h>
#include <sys/sbuf.h>

#include <assert.h>
#include <ctype.h>
#include <errno.h>
#include <stdbool.h>
#include <stdlib.h>
#include <string.h>
#include <yaml.h>

#include "pkg.h"
#include "private/event.h"
#include "private/pkg.h"
#include "private/utils.h"

#define PKG_UNKNOWN -1
#define PKG_DEPS -2
#define PKG_FILES -3
#define PKG_DIRS -4
#define PKG_SCRIPTS -5
#define PKG_CATEGORIES -6
#define PKG_LICENSES -7
#define PKG_OPTIONS -8
#define PKG_USERS -9
#define PKG_GROUPS -10
#define PKG_DIRECTORIES -11
#define PKG_SHLIBS -12

static int pkg_set_from_node(struct pkg *, yaml_node_t *, yaml_document_t *, int);
static int pkg_set_flatsize_from_node(struct pkg *, yaml_node_t *, yaml_document_t *, int);
static int pkg_set_licenselogic_from_node(struct pkg *, yaml_node_t *, yaml_document_t *, int);
static int pkg_set_deps_from_node(struct pkg *, yaml_node_t *, yaml_document_t *, const char *);
static int pkg_set_files_from_node(struct pkg *, yaml_node_t *, yaml_document_t *, const char *);
static int pkg_set_dirs_from_node(struct pkg *, yaml_node_t *, yaml_document_t *, const char *);
static int parse_sequence(struct pkg *, yaml_node_t *, yaml_document_t *, int);
static int parse_mapping(struct pkg *, yaml_node_t *, yaml_document_t *, int);

static struct manifest_key {
	const char *key;
	int type;
	yaml_node_type_t valid_type;
	int (*parse_data)(struct pkg *, yaml_node_t *, yaml_document_t *, int);
} manifest_key[] = {
	{ "name", PKG_NAME, YAML_SCALAR_NODE, pkg_set_from_node},
	{ "origin", PKG_ORIGIN, YAML_SCALAR_NODE, pkg_set_from_node},
	{ "version", PKG_VERSION, YAML_SCALAR_NODE, pkg_set_from_node},
	{ "arch", PKG_ARCH, YAML_SCALAR_NODE, pkg_set_from_node},
	{ "www", PKG_WWW, YAML_SCALAR_NODE, pkg_set_from_node},
	{ "comment", PKG_COMMENT, YAML_SCALAR_NODE, pkg_set_from_node},
	{ "maintainer", PKG_MAINTAINER, YAML_SCALAR_NODE, pkg_set_from_node},
	{ "prefix", PKG_PREFIX, YAML_SCALAR_NODE, pkg_set_from_node},
	{ "deps", PKG_DEPS, YAML_MAPPING_NODE, parse_mapping},
	{ "files", PKG_FILES, YAML_MAPPING_NODE, parse_mapping},
	{ "dirs", PKG_DIRS, YAML_SEQUENCE_NODE, parse_sequence},
	{ "directories", PKG_DIRECTORIES, YAML_MAPPING_NODE, parse_mapping},
	{ "flatsize", -1, YAML_SCALAR_NODE, pkg_set_flatsize_from_node},
	{ "licenselogic", -1, YAML_SCALAR_NODE, pkg_set_licenselogic_from_node},
	{ "licenses", PKG_LICENSES, YAML_SEQUENCE_NODE, parse_sequence},
	{ "desc", PKG_DESC, YAML_SCALAR_NODE, pkg_set_from_node},
	{ "scripts", PKG_SCRIPTS, YAML_MAPPING_NODE, parse_mapping},
	{ "message", PKG_MESSAGE, YAML_SCALAR_NODE, pkg_set_from_node},
	{ "infos", PKG_INFOS, YAML_SCALAR_NODE, pkg_set_from_node},
	{ "categories", PKG_CATEGORIES, YAML_SEQUENCE_NODE, parse_sequence},
	{ "options", PKG_OPTIONS, YAML_MAPPING_NODE, parse_mapping},
	/* compatibility with old format */
	{ "users", PKG_USERS, YAML_SEQUENCE_NODE, parse_sequence},
	{ "users", PKG_USERS, YAML_MAPPING_NODE, parse_mapping},
	{ "groups", PKG_GROUPS, YAML_SEQUENCE_NODE, parse_sequence},
	/* compatibility with old format */
	{ "groups", PKG_GROUPS, YAML_MAPPING_NODE, parse_mapping},
	{ "shlibs", PKG_SHLIBS, YAML_SEQUENCE_NODE, parse_sequence},
	{ NULL, -99, -99, NULL}
};

static int
is_valid_yaml_scalar(yaml_node_t *val)
{
	return (val->type == YAML_SCALAR_NODE && val->data.scalar.length > 0);
}

static int
urlencode(const char *src, struct sbuf **dest)
{
	size_t len;
	size_t i;

	sbuf_init(dest);

	len = strlen(src);
	for (i = 0; i < len; i++) {
		if (!isascii(src[i]) || src[i] == '%')
			sbuf_printf(*dest, "%%%.2x", (unsigned char)src[i]);
		else
			sbuf_putc(*dest, src[i]);
	}
	sbuf_finish(*dest);

	return (EPKG_OK);
}


static int
urldecode(const char *src, struct sbuf **dest)
{
	size_t len;
	size_t i;
	char c;
	char hex[] = {'\0', '\0', '\0'};

	sbuf_init(dest);

	len = strlen(src);
	for (i = 0; i < len; i++) {
		if (src[i] != '%') {
			sbuf_putc(*dest, src[i]);
		} else {
			if (i + 2 > len) {
				pkg_emit_error("unexpected end of string");
				return (EPKG_FATAL);
			}

			hex[0] = src[++i];
			hex[1] = src[++i];
			errno = 0;
			c = strtol(hex, NULL, 16);
			if (errno != 0) {
				/*
				 * if it fails consider this is not a urlencoded
				 * information
				 */
				sbuf_printf(*dest, "%%%s", hex);
			} else {
				sbuf_putc(*dest, c);
			}
		}
	}
	sbuf_finish(*dest);

	return (EPKG_OK);
}

int
pkg_load_manifest_file(struct pkg *pkg, const char *fpath)
{
	char *manifest = NULL;
	off_t sz;
	int ret = EPKG_OK;

	if ((ret = file_to_buffer(fpath, &manifest, &sz)) != EPKG_OK)
		return (ret);

	ret = pkg_parse_manifest(pkg, manifest);
	free(manifest);

	return (ret);
}

static int
pkg_set_from_node(struct pkg *pkg, yaml_node_t *val,
    __unused yaml_document_t *doc, int attr)
{
	int ret = EPKG_OK;

	while (val->data.scalar.length > 0 &&
	    val->data.scalar.value[val->data.scalar.length - 1] == '\n') {
		val->data.scalar.value[val->data.scalar.length - 1] = '\0';
		val->data.scalar.length--;
	}

	ret = urldecode(val->data.scalar.value, &pkg->fields[attr]);

	return (ret);
}

static int
pkg_set_flatsize_from_node(struct pkg *pkg, yaml_node_t *val,
    __unused yaml_document_t *doc, __unused int attr)
{
	int64_t flatsize;
	const char *errstr = NULL;
	flatsize = strtonum(val->data.scalar.value, 0, INT64_MAX, &errstr);
	if (errstr) {
		pkg_emit_error("Unable to convert %s to int64: %s",
					   val->data.scalar.value, errstr);
		return (EPKG_FATAL);
	}

	return (pkg_set(pkg, PKG_FLATSIZE, flatsize));
}
static int
pkg_set_licenselogic_from_node(struct pkg *pkg, yaml_node_t *val,
    __unused yaml_document_t *doc, __unused int attr)
{
	if (!strcmp(val->data.scalar.value, "single"))
		pkg_set(pkg, PKG_LICENSE_LOGIC, (int64_t) LICENSE_SINGLE);
	else if (!strcmp(val->data.scalar.value, "and") ||
	    !strcmp(val->data.scalar.value, "dual"))
		pkg_set(pkg, PKG_LICENSE_LOGIC, (int64_t)LICENSE_AND);
	else if (!strcmp(val->data.scalar.value, "or") ||
	    !strcmp(val->data.scalar.value, "multi"))
		pkg_set(pkg, PKG_LICENSE_LOGIC, (int64_t)LICENSE_OR);
	else {
		pkg_emit_error("Unknown license logic: %s",
		    val->data.scalar.value);
		return (EPKG_FATAL);
	}
	return (EPKG_OK);
}

static int
parse_sequence(struct pkg * pkg, yaml_node_t *node, yaml_document_t *doc,
    int attr)
{
	yaml_node_item_t *item;
	yaml_node_t *val;

	item = node->data.sequence.items.start;
	while (item < node->data.sequence.items.top) {
		val = yaml_document_get_node(doc, *item);
		switch (attr) {
		case PKG_CATEGORIES:
			if (!is_valid_yaml_scalar(val))
				pkg_emit_error("Skipping malformed category");
			else
				pkg_addcategory(pkg, val->data.scalar.value);
			break;
		case PKG_LICENSES:
			if (!is_valid_yaml_scalar(val))
				pkg_emit_error("Skipping malformed license");
			else
				pkg_addlicense(pkg, val->data.scalar.value);
			break;
		case PKG_USERS:
			if (is_valid_yaml_scalar(val))
				pkg_adduser(pkg, val->data.scalar.value);
			else if (val->type == YAML_MAPPING_NODE)
				parse_mapping(pkg, val, doc, attr);
			else
				pkg_emit_error("Skipping malformed license");
			break;
		case PKG_GROUPS:
			if (is_valid_yaml_scalar(val))
				pkg_addgroup(pkg, val->data.scalar.value);
			else if (val->type == YAML_MAPPING_NODE)
				parse_mapping(pkg, val, doc, attr);
			else
				pkg_emit_error("Skipping malformed license");
			break;
		case PKG_DIRS:
			if (is_valid_yaml_scalar(val))
				pkg_adddir(pkg, val->data.scalar.value, 1);
			else if (val->type == YAML_MAPPING_NODE)
				parse_mapping(pkg, val, doc, attr);
			else
				pkg_emit_error("Skipping malformed dirs");
			break;
		case PKG_SHLIBS:
			if (!is_valid_yaml_scalar(val))
				pkg_emit_error("Skipping malformed shared library");
			else
				pkg_addshlib(pkg, val->data.scalar.value);
		}
		++item;
	}
	return (EPKG_OK);
}

static int
script_type_str(const char *str)
{
	if (strcmp(str, "pre-install") == 0)
		return (PKG_SCRIPT_PRE_INSTALL);
	if (strcmp(str, "install") == 0)
		return (PKG_SCRIPT_INSTALL);
	if (strcmp(str, "post-install") == 0)
		return (PKG_SCRIPT_POST_INSTALL);
	if (strcmp(str, "pre-upgrade") == 0)
		return (PKG_SCRIPT_PRE_UPGRADE);
	if (strcmp(str, "upgrade") == 0)
		return (PKG_SCRIPT_UPGRADE);
	if (strcmp(str, "post-upgrade") == 0)
		return (PKG_SCRIPT_POST_UPGRADE);
	if (strcmp(str, "pre-deinstall") == 0)
		return (PKG_SCRIPT_PRE_DEINSTALL);
	if (strcmp(str, "deinstall") == 0)
		return (PKG_SCRIPT_DEINSTALL);
	if (strcmp(str, "post-deinstall") == 0)
		return (PKG_SCRIPT_POST_DEINSTALL);
	return (INT_MAX);
}

static int
parse_mapping(struct pkg *pkg, yaml_node_t *item, yaml_document_t *doc, int attr)
{
	struct sbuf *tmp = NULL;
	yaml_node_pair_t *pair;
	yaml_node_t *key;
	yaml_node_t *val;
	pkg_script script_type;

	pair = item->data.mapping.pairs.start;

	while (pair < item->data.mapping.pairs.top) {
		key = yaml_document_get_node(doc, pair->key);
		val = yaml_document_get_node(doc, pair->value);

		if (key->data.scalar.length <= 0) {
			pkg_emit_error("Skipping empty dependency name");
			++pair;
			continue;
		}

		switch (attr) {
		case PKG_DEPS:
			if (val->type != YAML_MAPPING_NODE)
				pkg_emit_error("Skipping malformed dependency %s",
				    key->data.scalar.value);
			else
				pkg_set_deps_from_node(pkg, val, doc,
				    key->data.scalar.value);
			break;
		case PKG_DIRS:
			if (val->type != YAML_MAPPING_NODE)
				pkg_emit_error("Skipping malformed dirs %s",
				    key->data.scalar.value);
			else
				pkg_set_dirs_from_node(pkg, val, doc,
				    key->data.scalar.value);
			break;
		case PKG_USERS:
			if (is_valid_yaml_scalar(val))
				pkg_adduid(pkg, key->data.scalar.value,
				    val->data.scalar.value);
			else
				pkg_emit_error("Skipping malformed users %s",
						key->data.scalar.value);
			break;
		case PKG_GROUPS:
			if (is_valid_yaml_scalar(val))
				pkg_addgid(pkg, key->data.scalar.value,
				    val->data.scalar.value);
			else
				pkg_emit_error("Skipping malformed groups %s",
						key->data.scalar.value);
			break;
		case PKG_DIRECTORIES:
			if (is_valid_yaml_scalar(val)) {
				urldecode(key->data.scalar.value, &tmp);
				if (val->data.scalar.value[0] == 'y')
					pkg_adddir(pkg, sbuf_get(tmp), 1);
				else
					pkg_adddir(pkg, sbuf_get(tmp), 0);
			} else if (val->type == YAML_MAPPING_NODE) {
				pkg_set_dirs_from_node(pkg, val, doc,
				    key->data.scalar.value);
			} else {
				pkg_emit_error("Skipping malformed directories %s",
				    key->data.scalar.value);
			}
			break;
		case PKG_FILES:
			if (is_valid_yaml_scalar(val)) {
				const char *pkg_sum = NULL;
				if (val->data.scalar.length == 64)
					pkg_sum = val->data.scalar.value;
				urldecode(key->data.scalar.value, &tmp);
				pkg_addfile(pkg, sbuf_get(tmp), pkg_sum, true);
			} else if (val->type == YAML_MAPPING_NODE)
				pkg_set_files_from_node(pkg, val, doc,
				    key->data.scalar.value);
			else
				pkg_emit_error("Skipping malformed files %s",
				    key->data.scalar.value);
			break;
		case PKG_OPTIONS:<commit_msg>Do not check from duplicate when loading from manifest considering the manifest for files has been produced by pkgng, and thus is clean.<commit_after>/*
 * Copyright (c) 2011-2012 <NAME> <<EMAIL>>
 * Copyright (c) 2011-2012 <NAME> <<EMAIL>>
 * All rights reserved.
 * 
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer
 *    in this position and unchanged.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include <sys/types.h>
#include <sys/sbuf.h>

#include <assert.h>
#include <ctype.h>
#include <errno.h>
#include <stdbool.h>
#include <stdlib.h>
#include <string.h>
#include <yaml.h>

#include "pkg.h"
#include "private/event.h"
#include "private/pkg.h"
#include "private/utils.h"

#define PKG_UNKNOWN -1
#define PKG_DEPS -2
#define PKG_FILES -3
#define PKG_DIRS -4
#define PKG_SCRIPTS -5
#define PKG_CATEGORIES -6
#define PKG_LICENSES -7
#define PKG_OPTIONS -8
#define PKG_USERS -9
#define PKG_GROUPS -10
#define PKG_DIRECTORIES -11
#define PKG_SHLIBS -12

static int pkg_set_from_node(struct pkg *, yaml_node_t *, yaml_document_t *, int);
static int pkg_set_flatsize_from_node(struct pkg *, yaml_node_t *, yaml_document_t *, int);
static int pkg_set_licenselogic_from_node(struct pkg *, yaml_node_t *, yaml_document_t *, int);
static int pkg_set_deps_from_node(struct pkg *, yaml_node_t *, yaml_document_t *, const char *);
static int pkg_set_files_from_node(struct pkg *, yaml_node_t *, yaml_document_t *, const char *);
static int pkg_set_dirs_from_node(struct pkg *, yaml_node_t *, yaml_document_t *, const char *);
static int parse_sequence(struct pkg *, yaml_node_t *, yaml_document_t *, int);
static int parse_mapping(struct pkg *, yaml_node_t *, yaml_document_t *, int);

static struct manifest_key {
	const char *key;
	int type;
	yaml_node_type_t valid_type;
	int (*parse_data)(struct pkg *, yaml_node_t *, yaml_document_t *, int);
} manifest_key[] = {
	{ "name", PKG_NAME, YAML_SCALAR_NODE, pkg_set_from_node},
	{ "origin", PKG_ORIGIN, YAML_SCALAR_NODE, pkg_set_from_node},
	{ "version", PKG_VERSION, YAML_SCALAR_NODE, pkg_set_from_node},
	{ "arch", PKG_ARCH, YAML_SCALAR_NODE, pkg_set_from_node},
	{ "www", PKG_WWW, YAML_SCALAR_NODE, pkg_set_from_node},
	{ "comment", PKG_COMMENT, YAML_SCALAR_NODE, pkg_set_from_node},
	{ "maintainer", PKG_MAINTAINER, YAML_SCALAR_NODE, pkg_set_from_node},
	{ "prefix", PKG_PREFIX, YAML_SCALAR_NODE, pkg_set_from_node},
	{ "deps", PKG_DEPS, YAML_MAPPING_NODE, parse_mapping},
	{ "files", PKG_FILES, YAML_MAPPING_NODE, parse_mapping},
	{ "dirs", PKG_DIRS, YAML_SEQUENCE_NODE, parse_sequence},
	{ "directories", PKG_DIRECTORIES, YAML_MAPPING_NODE, parse_mapping},
	{ "flatsize", -1, YAML_SCALAR_NODE, pkg_set_flatsize_from_node},
	{ "licenselogic", -1, YAML_SCALAR_NODE, pkg_set_licenselogic_from_node},
	{ "licenses", PKG_LICENSES, YAML_SEQUENCE_NODE, parse_sequence},
	{ "desc", PKG_DESC, YAML_SCALAR_NODE, pkg_set_from_node},
	{ "scripts", PKG_SCRIPTS, YAML_MAPPING_NODE, parse_mapping},
	{ "message", PKG_MESSAGE, YAML_SCALAR_NODE, pkg_set_from_node},
	{ "infos", PKG_INFOS, YAML_SCALAR_NODE, pkg_set_from_node},
	{ "categories", PKG_CATEGORIES, YAML_SEQUENCE_NODE, parse_sequence},
	{ "options", PKG_OPTIONS, YAML_MAPPING_NODE, parse_mapping},
	/* compatibility with old format */
	{ "users", PKG_USERS, YAML_SEQUENCE_NODE, parse_sequence},
	{ "users", PKG_USERS, YAML_MAPPING_NODE, parse_mapping},
	{ "groups", PKG_GROUPS, YAML_SEQUENCE_NODE, parse_sequence},
	/* compatibility with old format */
	{ "groups", PKG_GROUPS, YAML_MAPPING_NODE, parse_mapping},
	{ "shlibs", PKG_SHLIBS, YAML_SEQUENCE_NODE, parse_sequence},
	{ NULL, -99, -99, NULL}
};

static int
is_valid_yaml_scalar(yaml_node_t *val)
{
	return (val->type == YAML_SCALAR_NODE && val->data.scalar.length > 0);
}

static int
urlencode(const char *src, struct sbuf **dest)
{
	size_t len;
	size_t i;

	sbuf_init(dest);

	len = strlen(src);
	for (i = 0; i < len; i++) {
		if (!isascii(src[i]) || src[i] == '%')
			sbuf_printf(*dest, "%%%.2x", (unsigned char)src[i]);
		else
			sbuf_putc(*dest, src[i]);
	}
	sbuf_finish(*dest);

	return (EPKG_OK);
}


static int
urldecode(const char *src, struct sbuf **dest)
{
	size_t len;
	size_t i;
	char c;
	char hex[] = {'\0', '\0', '\0'};

	sbuf_init(dest);

	len = strlen(src);
	for (i = 0; i < len; i++) {
		if (src[i] != '%') {
			sbuf_putc(*dest, src[i]);
		} else {
			if (i + 2 > len) {
				pkg_emit_error("unexpected end of string");
				return (EPKG_FATAL);
			}

			hex[0] = src[++i];
			hex[1] = src[++i];
			errno = 0;
			c = strtol(hex, NULL, 16);
			if (errno != 0) {
				/*
				 * if it fails consider this is not a urlencoded
				 * information
				 */
				sbuf_printf(*dest, "%%%s", hex);
			} else {
				sbuf_putc(*dest, c);
			}
		}
	}
	sbuf_finish(*dest);

	return (EPKG_OK);
}

int
pkg_load_manifest_file(struct pkg *pkg, const char *fpath)
{
	char *manifest = NULL;
	off_t sz;
	int ret = EPKG_OK;

	if ((ret = file_to_buffer(fpath, &manifest, &sz)) != EPKG_OK)
		return (ret);

	ret = pkg_parse_manifest(pkg, manifest);
	free(manifest);

	return (ret);
}

static int
pkg_set_from_node(struct pkg *pkg, yaml_node_t *val,
    __unused yaml_document_t *doc, int attr)
{
	int ret = EPKG_OK;

	while (val->data.scalar.length > 0 &&
	    val->data.scalar.value[val->data.scalar.length - 1] == '\n') {
		val->data.scalar.value[val->data.scalar.length - 1] = '\0';
		val->data.scalar.length--;
	}

	ret = urldecode(val->data.scalar.value, &pkg->fields[attr]);

	return (ret);
}

static int
pkg_set_flatsize_from_node(struct pkg *pkg, yaml_node_t *val,
    __unused yaml_document_t *doc, __unused int attr)
{
	int64_t flatsize;
	const char *errstr = NULL;
	flatsize = strtonum(val->data.scalar.value, 0, INT64_MAX, &errstr);
	if (errstr) {
		pkg_emit_error("Unable to convert %s to int64: %s",
					   val->data.scalar.value, errstr);
		return (EPKG_FATAL);
	}

	return (pkg_set(pkg, PKG_FLATSIZE, flatsize));
}
static int
pkg_set_licenselogic_from_node(struct pkg *pkg, yaml_node_t *val,
    __unused yaml_document_t *doc, __unused int attr)
{
	if (!strcmp(val->data.scalar.value, "single"))
		pkg_set(pkg, PKG_LICENSE_LOGIC, (int64_t) LICENSE_SINGLE);
	else if (!strcmp(val->data.scalar.value, "and") ||
	    !strcmp(val->data.scalar.value, "dual"))
		pkg_set(pkg, PKG_LICENSE_LOGIC, (int64_t)LICENSE_AND);
	else if (!strcmp(val->data.scalar.value, "or") ||
	    !strcmp(val->data.scalar.value, "multi"))
		pkg_set(pkg, PKG_LICENSE_LOGIC, (int64_t)LICENSE_OR);
	else {
		pkg_emit_error("Unknown license logic: %s",
		    val->data.scalar.value);
		return (EPKG_FATAL);
	}
	return (EPKG_OK);
}

static int
parse_sequence(struct pkg * pkg, yaml_node_t *node, yaml_document_t *doc,
    int attr)
{
	yaml_node_item_t *item;
	yaml_node_t *val;

	item = node->data.sequence.items.start;
	while (item < node->data.sequence.items.top) {
		val = yaml_document_get_node(doc, *item);
		switch (attr) {
		case PKG_CATEGORIES:
			if (!is_valid_yaml_scalar(val))
				pkg_emit_error("Skipping malformed category");
			else
				pkg_addcategory(pkg, val->data.scalar.value);
			break;
		case PKG_LICENSES:
			if (!is_valid_yaml_scalar(val))
				pkg_emit_error("Skipping malformed license");
			else
				pkg_addlicense(pkg, val->data.scalar.value);
			break;
		case PKG_USERS:
			if (is_valid_yaml_scalar(val))
				pkg_adduser(pkg, val->data.scalar.value);
			else if (val->type == YAML_MAPPING_NODE)
				parse_mapping(pkg, val, doc, attr);
			else
				pkg_emit_error("Skipping malformed license");
			break;
		case PKG_GROUPS:
			if (is_valid_yaml_scalar(val))
				pkg_addgroup(pkg, val->data.scalar.value);
			else if (val->type == YAML_MAPPING_NODE)
				parse_mapping(pkg, val, doc, attr);
			else
				pkg_emit_error("Skipping malformed license");
			break;
		case PKG_DIRS:
			if (is_valid_yaml_scalar(val))
				pkg_adddir(pkg, val->data.scalar.value, 1);
			else if (val->type == YAML_MAPPING_NODE)
				parse_mapping(pkg, val, doc, attr);
			else
				pkg_emit_error("Skipping malformed dirs");
			break;
		case PKG_SHLIBS:
			if (!is_valid_yaml_scalar(val))
				pkg_emit_error("Skipping malformed shared library");
			else
				pkg_addshlib(pkg, val->data.scalar.value);
		}
		++item;
	}
	return (EPKG_OK);
}

static int
script_type_str(const char *str)
{
	if (strcmp(str, "pre-install") == 0)
		return (PKG_SCRIPT_PRE_INSTALL);
	if (strcmp(str, "install") == 0)
		return (PKG_SCRIPT_INSTALL);
	if (strcmp(str, "post-install") == 0)
		return (PKG_SCRIPT_POST_INSTALL);
	if (strcmp(str, "pre-upgrade") == 0)
		return (PKG_SCRIPT_PRE_UPGRADE);
	if (strcmp(str, "upgrade") == 0)
		return (PKG_SCRIPT_UPGRADE);
	if (strcmp(str, "post-upgrade") == 0)
		return (PKG_SCRIPT_POST_UPGRADE);
	if (strcmp(str, "pre-deinstall") == 0)
		return (PKG_SCRIPT_PRE_DEINSTALL);
	if (strcmp(str, "deinstall") == 0)
		return (PKG_SCRIPT_DEINSTALL);
	if (strcmp(str, "post-deinstall") == 0)
		return (PKG_SCRIPT_POST_DEINSTALL);
	return (INT_MAX);
}

static int
parse_mapping(struct pkg *pkg, yaml_node_t *item, yaml_document_t *doc, int attr)
{
	struct sbuf *tmp = NULL;
	yaml_node_pair_t *pair;
	yaml_node_t *key;
	yaml_node_t *val;
	pkg_script script_type;

	pair = item->data.mapping.pairs.start;

	while (pair < item->data.mapping.pairs.top) {
		key = yaml_document_get_node(doc, pair->key);
		val = yaml_document_get_node(doc, pair->value);

		if (key->data.scalar.length <= 0) {
			pkg_emit_error("Skipping empty dependency name");
			++pair;
			continue;
		}

		switch (attr) {
		case PKG_DEPS:
			if (val->type != YAML_MAPPING_NODE)
				pkg_emit_error("Skipping malformed dependency %s",
				    key->data.scalar.value);
			else
				pkg_set_deps_from_node(pkg, val, doc,
				    key->data.scalar.value);
			break;
		case PKG_DIRS:
			if (val->type != YAML_MAPPING_NODE)
				pkg_emit_error("Skipping malformed dirs %s",
				    key->data.scalar.value);
			else
				pkg_set_dirs_from_node(pkg, val, doc,
				    key->data.scalar.value);
			break;
		case PKG_USERS:
			if (is_valid_yaml_scalar(val))
				pkg_adduid(pkg, key->data.scalar.value,
				    val->data.scalar.value);
			else
				pkg_emit_error("Skipping malformed users %s",
						key->data.scalar.value);
			break;
		case PKG_GROUPS:
			if (is_valid_yaml_scalar(val))
				pkg_addgid(pkg, key->data.scalar.value,
				    val->data.scalar.value);
			else
				pkg_emit_error("Skipping malformed groups %s",
						key->data.scalar.value);
			break;
		case PKG_DIRECTORIES:
			if (is_valid_yaml_scalar(val)) {
				urldecode(key->data.scalar.value, &tmp);
				if (val->data.scalar.value[0] == 'y')
					pkg_adddir(pkg, sbuf_get(tmp), 1);
				else
					pkg_adddir(pkg, sbuf_get(tmp), 0);
			} else if (val->type == YAML_MAPPING_NODE) {
				pkg_set_dirs_from_node(pkg, val, doc,
				    key->data.scalar.value);
			} else {
				pkg_emit_error("Skipping malformed directories %s",
				    key->data.scalar.value);
			}
			break;
		case PKG_FILES:
			if (is_valid_yaml_scalar(val)) {
				const char *pkg_sum = NULL;
				if (val->data.scalar.length == 64)
					pkg_sum = val->data.scalar.value;
				urldecode(key->data.scalar.value, &tmp);
				pkg_addfile(pkg, sbuf_get(tmp), pkg_sum, false);
			} else if (val->type == YAML_MAPPING_NODE)
				pkg_set_files_from_node(pkg, val, doc,
				    key->data.scalar.value);
			else
				pkg_emit_error("Skipping malformed files %s",
				    key->data.scalar.value);
			break;
		case PKG_OPTIONS:
{'fraction_non_alphanumeric': 0.13441249899331562, 'fraction_numerical': 0.006925988564065394, 'mean_word_length': 4.45344751866491, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 153, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1962975', 'n_tokens_mistral': 10614, 'n_tokens_neox': 9432, 'n_words': 2161}