<commit_before>/*-
 * Copyright (c) 2012 <NAME> <<EMAIL>>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer
 *    in this position and unchanged.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include <sys/stat.h>
#include <sys/param.h>
#include <sys/mman.h>

#define _WITH_GETLINE
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <dirent.h>
#include <errno.h>
#include <limits.h>

#include <archive.h>
#include <archive_entry.h>

#include "pkg.h"
#include "private/event.h"
#include "private/utils.h"
#include "private/pkgdb.h"
#include "private/repodb.h"

//#define EXTRACT_ARCHIVE_FLAGS  (ARCHIVE_EXTRACT_OWNER |ARCHIVE_EXTRACT_PERM)

struct sig_cert {
	char name[MAXPATHLEN];
	unsigned char *sig;
	int siglen;
	unsigned char *cert;
	int certlen;
	UT_hash_handle hh;
	bool trusted;
};

typedef enum {
	HASH_UNKNOWN,
	HASH_SHA256,
} hash_t;

struct fingerprint {
	hash_t type;
	char hash[BUFSIZ];
	UT_hash_handle hh;
};

/* Return opened file descriptor */
static int
repo_fetch_remote_tmp(struct pkg_repo *repo, const char *filename, const char *extension, time_t *t, int *rc)
{
	char url[MAXPATHLEN];
	char tmp[MAXPATHLEN];
	int fd;
	mode_t mask;
	const char *tmpdir;

	snprintf(url, MAXPATHLEN, "%s/%s.%s", pkg_repo_url(repo), filename, extension);

	tmpdir = getenv("TMPDIR");
	if (tmpdir == NULL)
		tmpdir = "/tmp";
	mkdirs(tmpdir);
	snprintf(tmp, MAXPATHLEN, "%s/%s.%s.XXXXXX", tmpdir, filename, extension);

	mask = umask(022);
	fd = mkstemp(tmp);
	umask(mask);
	if (fd == -1) {
		pkg_emit_error("Could not create temporary file %s, "
		    "aborting update.\n", tmp);
		*rc = EPKG_FATAL;
		return (-1);
	}
	(void)unlink(tmp);

	if ((*rc = pkg_fetch_file_to_fd(repo, url, fd, t)) != EPKG_OK) {
		close(fd);
		fd = -1;
	}

	return (fd);
}

static bool
has_ext(const char *path, const char *ext)
{
	size_t n, l;
	const char *p = NULL;

	n = strlen(path);
	l = strlen(ext);
	p = &path[n - l];

	if (strcmp(p, ext) == 0)
		return (true);

	return (false);
}

static struct fingerprint *
parse_fingerprint(ucl_object_t *obj)
{
	ucl_object_t *cur;
	ucl_object_iter_t it = NULL;
	const char *function = NULL, *fp = NULL;
	hash_t fct = HASH_UNKNOWN;
	struct fingerprint *f = NULL;
	const char *key;

	while ((cur = ucl_iterate_object(obj, &it, true))) {
		key = ucl_object_key(cur);
		if (cur->type != UCL_STRING)
			continue;

		if (strcasecmp(key, "function") == 0) {
			function = ucl_object_tostring(cur);
			continue;
		}

		if (strcasecmp(key, "fingerprint") == 0) {
			fp = ucl_object_tostring(cur);
			continue;
		}
	}

	if (fp == NULL || function == NULL)
		return (NULL);

	if (strcasecmp(function, "sha256") == 0)
		fct = HASH_SHA256;

	if (fct == HASH_UNKNOWN) {
		pkg_emit_error("Unsupported hashing function: %s", function);
		return (NULL);
	}

	f = calloc(1, sizeof(struct fingerprint));
	f->type = fct;
	strlcpy(f->hash, fp, sizeof(f->hash));

	return (f);
}

static struct fingerprint *
load_fingerprint(const char *dir, const char *filename)
{
	ucl_object_t *obj = NULL;
	struct ucl_parser *p = NULL;
	char path[MAXPATHLEN];
	struct fingerprint *f = NULL;

	snprintf(path, MAXPATHLEN, "%s/%s", dir, filename);

	p = ucl_parser_new(0);

	if (!ucl_parser_add_file(p, path)) {
		pkg_emit_error("%s", ucl_parser_get_error(p));
		ucl_parser_free(p);
		return (NULL);
	}

	obj = ucl_parser_get_object(p);

	if (obj->type == UCL_OBJECT)
		f = parse_fingerprint(obj);

	ucl_object_free(obj);
	ucl_parser_free(p);

	return (f);
}

static struct fingerprint *
load_fingerprints(const char *path)
{
	DIR *d;
	struct dirent *ent;
	struct fingerprint *f = NULL;
	struct fingerprint *finger = NULL;

	if ((d = opendir(path)) == NULL)
		return (NULL);

	while ((ent = readdir(d))) {
		if (strcmp(ent->d_name, ".") == 0 ||
		    strcmp(ent->d_name, "..") == 0)
			continue;
		finger = load_fingerprint(path, ent->d_name);
		if (finger != NULL)
			HASH_ADD_STR(f, hash, finger);
	}

	closedir(d);

	return (f);
}

static int
repo_archive_extract_file(int fd, const char *file, const char *dest, struct pkg_repo *repo, int dest_fd)
{
	struct archive *a = NULL;
	struct archive_entry *ae = NULL;
	struct sig_cert *sc = NULL;
	struct sig_cert *s = NULL, *stmp = NULL;
	struct fingerprint *trusted = NULL;
	struct fingerprint *revoked = NULL;
	struct fingerprint *f = NULL;
	unsigned char *sig = NULL;
	int siglen = 0, ret, rc = EPKG_OK;
	char key[MAXPATHLEN], path[MAXPATHLEN];
	char hash[SHA256_DIGEST_LENGTH * 2 + 1];
	int nbgood = 0;

	pkg_debug(1, "PkgRepo: extracting repo %", pkg_repo_name(repo));

	a = archive_read_new();
	archive_read_support_filter_all(a);
	archive_read_support_format_tar(a);

	/* Seek to the begin of file */
	(void)lseek(fd, 0, SEEK_SET);
	archive_read_open_fd(a, fd, 4096);

	while (archive_read_next_header(a, &ae) == ARCHIVE_OK) {
		if (strcmp(archive_entry_pathname(ae), file) == 0) {
			if (dest_fd == -1) {
				archive_entry_set_pathname(ae, dest);
				/*
				 * The repo should be owned by root and not writable
				 */
				archive_entry_set_uid(ae, 0);
				archive_entry_set_gid(ae, 0);
				archive_entry_set_perm(ae, 0644);

				if (archive_read_extract(a, ae, EXTRACT_ARCHIVE_FLAGS) != 0) {
					pkg_emit_errno("archive_read_extract", "extract error");
					rc = EPKG_FATAL;
					goto cleanup;
				}
			} else {
				if (archive_read_data_into_fd(a, dest_fd) != 0) {
					pkg_emit_errno("archive_read_extract", "extract error");
					rc = EPKG_FATAL;
					goto cleanup;
				}
				(void)lseek(dest_fd, 0, SEEK_SET);
			}
		}
		if (pkg_repo_signature_type(repo) == SIG_PUBKEY &&
		    strcmp(archive_entry_pathname(ae), "signature") == 0) {
			siglen = archive_entry_size(ae);
			sig = malloc(siglen);
			archive_read_data(a, sig, siglen);
		}

		if (pkg_repo_signature_type(repo) == SIG_FINGERPRINT) {
			if (has_ext(archive_entry_pathname(ae), ".sig")) {
				snprintf(key, MAXPATHLEN, "%.*s",
				    (int) strlen(archive_entry_pathname(ae)) - 4,
				    archive_entry_pathname(ae));
				HASH_FIND_STR(sc, key, s);
				if (s == NULL) {
					s = calloc(1, sizeof(struct sig_cert));
					strlcpy(s->name, key, MAXPATHLEN);
					HASH_ADD_STR(sc, name, s);
				}
				s->siglen = archive_entry_size(ae);
				s->sig = malloc(s->siglen);
				archive_read_data(a, s->sig, s->siglen);
			}
			if (has_ext(archive_entry_pathname(ae), ".cert")) {
				snprintf(key, MAXPATHLEN, "%.*s",
				    (int) strlen(archive_entry_pathname(ae)) - 5,
				    archive_entry_pathname(ae));
				HASH_FIND_STR(sc, key, s);
				if (s == NULL) {
					s = calloc(1, sizeof(struct sig_cert));
					strlcpy(s->name, key, MAXPATHLEN);
					HASH_ADD_STR(sc, name, s);
				}
				s->certlen = archive_entry_size(ae);
				s->cert = malloc(s->certlen);
				archive_read_data(a, s->cert, s->certlen);
			}
		}
	}

	if (pkg_repo_signature_type(repo) == SIG_PUBKEY) {
		if (sig == NULL) {
			pkg_emit_error("No signature found in the repository.  "
					"Can not validate against %s key.", pkg_repo_key(repo));
			rc = EPKG_FATAL;
			goto cleanup;
		}
		ret = rsa_verify(dest, pkg_repo_key(repo),
		    sig, siglen - 1, dest_fd);
		if (ret != EPKG_OK) {
			pkg_emit_error("Invalid signature, "
					"removing repository.");
			free(sig);
			rc = EPKG_FATAL;
			goto cleanup;
		}
		free(sig);
	} else if (pkg_repo_signature_type(repo) == SIG_FINGERPRINT) {
		if (HASH_COUNT(sc) == 0) {
			pkg_emit_error("No signature found");
			rc = EPKG_FATAL;
			goto cleanup;
		}

		/* load fingerprints */
		snprintf(path, MAXPATHLEN, "%s/trusted", pkg_repo_fingerprints(repo));
		if ((trusted = load_fingerprints(path)) == NULL) {
			pkg_emit_error("Error loading trusted certificates");
			rc = EPKG_FATAL;
			goto cleanup;
		}

		if (HASH_COUNT(trusted) == 0) {
			pkg_emit_error("No trusted certificates");
			rc = EPKG_FATAL;
			goto cleanup;
		}

		snprintf(path, MAXPATHLEN, "%s/revoked", pkg_repo_fingerprints(repo));
		if ((revoked = load_fingerprints(path)) == NULL) {
			pkg_emit_error("Error loading revoked certificates");<commit_msg>Fix erroring out when no revoked certs are found<commit_after>/*-
 * Copyright (c) 2012 <NAME> <<EMAIL>>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer
 *    in this position and unchanged.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include <sys/stat.h>
#include <sys/param.h>
#include <sys/mman.h>

#define _WITH_GETLINE
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <dirent.h>
#include <errno.h>
#include <limits.h>

#include <archive.h>
#include <archive_entry.h>

#include "pkg.h"
#include "private/event.h"
#include "private/utils.h"
#include "private/pkgdb.h"
#include "private/repodb.h"

//#define EXTRACT_ARCHIVE_FLAGS  (ARCHIVE_EXTRACT_OWNER |ARCHIVE_EXTRACT_PERM)

struct sig_cert {
	char name[MAXPATHLEN];
	unsigned char *sig;
	int siglen;
	unsigned char *cert;
	int certlen;
	UT_hash_handle hh;
	bool trusted;
};

typedef enum {
	HASH_UNKNOWN,
	HASH_SHA256,
} hash_t;

struct fingerprint {
	hash_t type;
	char hash[BUFSIZ];
	UT_hash_handle hh;
};

/* Return opened file descriptor */
static int
repo_fetch_remote_tmp(struct pkg_repo *repo, const char *filename, const char *extension, time_t *t, int *rc)
{
	char url[MAXPATHLEN];
	char tmp[MAXPATHLEN];
	int fd;
	mode_t mask;
	const char *tmpdir;

	snprintf(url, MAXPATHLEN, "%s/%s.%s", pkg_repo_url(repo), filename, extension);

	tmpdir = getenv("TMPDIR");
	if (tmpdir == NULL)
		tmpdir = "/tmp";
	mkdirs(tmpdir);
	snprintf(tmp, MAXPATHLEN, "%s/%s.%s.XXXXXX", tmpdir, filename, extension);

	mask = umask(022);
	fd = mkstemp(tmp);
	umask(mask);
	if (fd == -1) {
		pkg_emit_error("Could not create temporary file %s, "
		    "aborting update.\n", tmp);
		*rc = EPKG_FATAL;
		return (-1);
	}
	(void)unlink(tmp);

	if ((*rc = pkg_fetch_file_to_fd(repo, url, fd, t)) != EPKG_OK) {
		close(fd);
		fd = -1;
	}

	return (fd);
}

static bool
has_ext(const char *path, const char *ext)
{
	size_t n, l;
	const char *p = NULL;

	n = strlen(path);
	l = strlen(ext);
	p = &path[n - l];

	if (strcmp(p, ext) == 0)
		return (true);

	return (false);
}

static struct fingerprint *
parse_fingerprint(ucl_object_t *obj)
{
	ucl_object_t *cur;
	ucl_object_iter_t it = NULL;
	const char *function = NULL, *fp = NULL;
	hash_t fct = HASH_UNKNOWN;
	struct fingerprint *f = NULL;
	const char *key;

	while ((cur = ucl_iterate_object(obj, &it, true))) {
		key = ucl_object_key(cur);
		if (cur->type != UCL_STRING)
			continue;

		if (strcasecmp(key, "function") == 0) {
			function = ucl_object_tostring(cur);
			continue;
		}

		if (strcasecmp(key, "fingerprint") == 0) {
			fp = ucl_object_tostring(cur);
			continue;
		}
	}

	if (fp == NULL || function == NULL)
		return (NULL);

	if (strcasecmp(function, "sha256") == 0)
		fct = HASH_SHA256;

	if (fct == HASH_UNKNOWN) {
		pkg_emit_error("Unsupported hashing function: %s", function);
		return (NULL);
	}

	f = calloc(1, sizeof(struct fingerprint));
	f->type = fct;
	strlcpy(f->hash, fp, sizeof(f->hash));

	return (f);
}

static struct fingerprint *
load_fingerprint(const char *dir, const char *filename)
{
	ucl_object_t *obj = NULL;
	struct ucl_parser *p = NULL;
	char path[MAXPATHLEN];
	struct fingerprint *f = NULL;

	snprintf(path, MAXPATHLEN, "%s/%s", dir, filename);

	p = ucl_parser_new(0);

	if (!ucl_parser_add_file(p, path)) {
		pkg_emit_error("%s", ucl_parser_get_error(p));
		ucl_parser_free(p);
		return (NULL);
	}

	obj = ucl_parser_get_object(p);

	if (obj->type == UCL_OBJECT)
		f = parse_fingerprint(obj);

	ucl_object_free(obj);
	ucl_parser_free(p);

	return (f);
}

static int
load_fingerprints(const char *path, struct fingerprint **f)
{
	DIR *d;
	struct dirent *ent;
	struct fingerprint *finger = NULL;

	*f = NULL;

	if ((d = opendir(path)) == NULL)
		return (-1);

	while ((ent = readdir(d))) {
		if (strcmp(ent->d_name, ".") == 0 ||
		    strcmp(ent->d_name, "..") == 0)
			continue;
		finger = load_fingerprint(path, ent->d_name);
		if (finger != NULL)
			HASH_ADD_STR(*f, hash, finger);
	}

	closedir(d);

	return (0);
}

static int
repo_archive_extract_file(int fd, const char *file, const char *dest, struct pkg_repo *repo, int dest_fd)
{
	struct archive *a = NULL;
	struct archive_entry *ae = NULL;
	struct sig_cert *sc = NULL;
	struct sig_cert *s = NULL, *stmp = NULL;
	struct fingerprint *trusted = NULL;
	struct fingerprint *revoked = NULL;
	struct fingerprint *f = NULL;
	unsigned char *sig = NULL;
	int siglen = 0, ret, rc = EPKG_OK;
	char key[MAXPATHLEN], path[MAXPATHLEN];
	char hash[SHA256_DIGEST_LENGTH * 2 + 1];
	int nbgood = 0;

	pkg_debug(1, "PkgRepo: extracting repo %", pkg_repo_name(repo));

	a = archive_read_new();
	archive_read_support_filter_all(a);
	archive_read_support_format_tar(a);

	/* Seek to the begin of file */
	(void)lseek(fd, 0, SEEK_SET);
	archive_read_open_fd(a, fd, 4096);

	while (archive_read_next_header(a, &ae) == ARCHIVE_OK) {
		if (strcmp(archive_entry_pathname(ae), file) == 0) {
			if (dest_fd == -1) {
				archive_entry_set_pathname(ae, dest);
				/*
				 * The repo should be owned by root and not writable
				 */
				archive_entry_set_uid(ae, 0);
				archive_entry_set_gid(ae, 0);
				archive_entry_set_perm(ae, 0644);

				if (archive_read_extract(a, ae, EXTRACT_ARCHIVE_FLAGS) != 0) {
					pkg_emit_errno("archive_read_extract", "extract error");
					rc = EPKG_FATAL;
					goto cleanup;
				}
			} else {
				if (archive_read_data_into_fd(a, dest_fd) != 0) {
					pkg_emit_errno("archive_read_extract", "extract error");
					rc = EPKG_FATAL;
					goto cleanup;
				}
				(void)lseek(dest_fd, 0, SEEK_SET);
			}
		}
		if (pkg_repo_signature_type(repo) == SIG_PUBKEY &&
		    strcmp(archive_entry_pathname(ae), "signature") == 0) {
			siglen = archive_entry_size(ae);
			sig = malloc(siglen);
			archive_read_data(a, sig, siglen);
		}

		if (pkg_repo_signature_type(repo) == SIG_FINGERPRINT) {
			if (has_ext(archive_entry_pathname(ae), ".sig")) {
				snprintf(key, MAXPATHLEN, "%.*s",
				    (int) strlen(archive_entry_pathname(ae)) - 4,
				    archive_entry_pathname(ae));
				HASH_FIND_STR(sc, key, s);
				if (s == NULL) {
					s = calloc(1, sizeof(struct sig_cert));
					strlcpy(s->name, key, MAXPATHLEN);
					HASH_ADD_STR(sc, name, s);
				}
				s->siglen = archive_entry_size(ae);
				s->sig = malloc(s->siglen);
				archive_read_data(a, s->sig, s->siglen);
			}
			if (has_ext(archive_entry_pathname(ae), ".cert")) {
				snprintf(key, MAXPATHLEN, "%.*s",
				    (int) strlen(archive_entry_pathname(ae)) - 5,
				    archive_entry_pathname(ae));
				HASH_FIND_STR(sc, key, s);
				if (s == NULL) {
					s = calloc(1, sizeof(struct sig_cert));
					strlcpy(s->name, key, MAXPATHLEN);
					HASH_ADD_STR(sc, name, s);
				}
				s->certlen = archive_entry_size(ae);
				s->cert = malloc(s->certlen);
				archive_read_data(a, s->cert, s->certlen);
			}
		}
	}

	if (pkg_repo_signature_type(repo) == SIG_PUBKEY) {
		if (sig == NULL) {
			pkg_emit_error("No signature found in the repository.  "
					"Can not validate against %s key.", pkg_repo_key(repo));
			rc = EPKG_FATAL;
			goto cleanup;
		}
		ret = rsa_verify(dest, pkg_repo_key(repo),
		    sig, siglen - 1, dest_fd);
		if (ret != EPKG_OK) {
			pkg_emit_error("Invalid signature, "
					"removing repository.");
			free(sig);
			rc = EPKG_FATAL;
			goto cleanup;
		}
		free(sig);
	} else if (pkg_repo_signature_type(repo) == SIG_FINGERPRINT) {
		if (HASH_COUNT(sc) == 0) {
			pkg_emit_error("No signature found");
			rc = EPKG_FATAL;
			goto cleanup;
		}

		/* load fingerprints */
		snprintf(path, MAXPATHLEN, "%s/trusted", pkg_repo_fingerprints(repo));
		if ((load_fingerprints(path, &trusted)) == -1) {
			pkg_emit_error("Error loading trusted certificates");
			rc = EPKG_FATAL;
			goto cleanup;
		}

		if (HASH_COUNT(trusted) == 0) {
			pkg_emit_error("No trusted certificates");
			rc = EPKG_FATAL;
			goto cleanup;
		}

		snprintf(path, MAXPATHLEN, "%s/revoked", pkg_repo_fingerprints(repo));
		if ((load_fingerprints(path, &revoked)) == -1) {
			pkg_emit_error("Error loading revoked certificates");
{'fraction_non_alphanumeric': 0.13697584383008887, 'fraction_numerical': 0.006870603631604777, 'mean_word_length': 4.057915057915058, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1149453', 'n_tokens_mistral': 7651, 'n_tokens_neox': 6729, 'n_words': 1763}