<commit_before>/*-
 * Copyright (c) 2011-2013 <NAME> <<EMAIL>>
 * Copyright (c) 2011-2012 <NAME> <<EMAIL>>
 * Copyright (c) 2011-2012 <NAME> <<EMAIL>>
 * Copyright (c) 2012-2013 <NAME> <<EMAIL>>
 * Copyright (c) 2014 <NAME> <<EMAIL>>
 *
 * All rights reserved.
 * 
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer
 *    in this position and unchanged.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include <sys/types.h>
#include <sys/stat.h>
#include <sys/sysctl.h>
#include <sys/uio.h>

#include <archive_entry.h>
#include <assert.h>
#include <fts.h>
#include <libgen.h>
#include <sqlite3.h>
#include <string.h>
#include <dirent.h>
#define _WITH_GETLINE
#include <stdio.h>
#include <stdbool.h>
#include <sysexits.h>
#include <unistd.h>
#include <errno.h>
#include <sys/mman.h>
#include <fcntl.h>

#include "pkg.h"
#include "private/event.h"
#include "private/utils.h"
#include "private/pkg.h"
#include "private/pkgdb.h"
#include "private/repodb.h"
#include "private/thd_repo.h"


struct sig_cert {
	char name[MAXPATHLEN];
	unsigned char *sig;
	int64_t siglen;
	unsigned char *cert;
	int64_t certlen;
	bool cert_allocated;
	UT_hash_handle hh;
	bool trusted;
};

void
pkg_repo_cached_name(struct pkg *pkg, char *dest, size_t destlen)
{
	const char *sum, *name, *version, *reponame, *repourl, *ext = NULL;
	const char *cachedir = NULL;
	struct stat st;

	cachedir = pkg_object_string(pkg_config_get("PKG_CACHEDIR"));

	pkg_get(pkg, PKG_REPONAME, &reponame,
			PKG_CKSUM, &sum, PKG_NAME, &name, PKG_VERSION, &version,
			PKG_REPOPATH, &repourl);

	if (repourl != NULL)
		ext = strrchr(repourl, '.');

	if (ext != NULL) {
		/*
		 * XXX:
		 * This code tries to skip refetching but it should be removed as soon
		 * as we transfer to new scheme.
		 */
		pkg_snprintf(dest, destlen, "%S/%n-%v-%z",
				cachedir, pkg, pkg, pkg);
		if (stat (dest, &st) != -1)
			return;

		/*
		 * The real naming scheme:
		 * <cachedir>/<name>-<version>-<checksum>.txz
		 */
		pkg_snprintf(dest, destlen, "%S/%n-%v-%z%S",
				cachedir, pkg, pkg, pkg, ext);
	}
	else {
		pkg_snprintf(dest, destlen, "%S/%n-%v-%z",
				cachedir, pkg, pkg, pkg);
	}
}

int
pkg_repo_fetch_package(struct pkg *pkg)
{
	char dest[MAXPATHLEN];
	char url[MAXPATHLEN];
	int fetched = 0;
	char cksum[SHA256_DIGEST_LENGTH * 2 +1];
	char *path = NULL;
	const char *packagesite = NULL;

	int retcode = EPKG_OK;
	const char *reponame, *name, *version, *sum;
	struct pkg_repo *repo;

	assert((pkg->type & PKG_REMOTE) == PKG_REMOTE);

	pkg_get(pkg, PKG_REPONAME, &reponame,
			PKG_CKSUM, &sum, PKG_NAME, &name, PKG_VERSION, &version);
	pkg_repo_cached_name(pkg, dest, sizeof(dest));

	/* If it is already in the local cachedir, dont bother to
	 * download it */
	if (access(dest, F_OK) == 0)
		goto checksum;

	/* Create the dirs in cachedir */
	if ((path = dirname(dest)) == NULL) {
		pkg_emit_errno("dirname", dest);
		retcode = EPKG_FATAL;
		goto cleanup;
	}

	if ((retcode = mkdirs(path)) != EPKG_OK)
		goto cleanup;

	/*
	 * In multi-repos the remote URL is stored in pkg[PKG_REPOURL]
	 * For a single attached database the repository URL should be
	 * defined by PACKAGESITE.
	 */
	repo = pkg_repo_find_name(reponame);
	packagesite = pkg_repo_url(repo);

	if (packagesite == NULL || packagesite[0] == '\0') {
		pkg_emit_error("PACKAGESITE is not defined");
		retcode = 1;
		goto cleanup;
	}

	if (packagesite[strlen(packagesite) - 1] == '/')
		pkg_snprintf(url, sizeof(url), "%S%R", packagesite, pkg);
	else
		pkg_snprintf(url, sizeof(url), "%S/%R", packagesite, pkg);

	if (strncasecmp(packagesite, "file://", 7) == 0) {
		pkg_set(pkg, PKG_REPOPATH, url + 7);
		return (EPKG_OK);
	}

	retcode = pkg_fetch_file(repo, url, dest, 0);
	fetched = 1;

	if (retcode != EPKG_OK)
		goto cleanup;

	checksum:
	retcode = sha256_file(dest, cksum);
	if (retcode == EPKG_OK)
		if (strcmp(cksum, sum)) {
			if (fetched == 1) {
				pkg_emit_error("%s-%s failed checksum "
				    "from repository", name, version);
				retcode = EPKG_FATAL;
			} else {
				pkg_emit_error("cached package %s-%s: "
				    "checksum mismatch, fetching from remote",
				    name, version);
				unlink(dest);
				return (pkg_repo_fetch_package(pkg));
			}
		}

	cleanup:
	if (retcode != EPKG_OK)
		unlink(dest);

	return (retcode);
}

static int
pkg_repo_fetch_remote_tmp(struct pkg_repo *repo,
		const char *filename, const char *extension, time_t *t, int *rc)
{
	char url[MAXPATHLEN];
	char tmp[MAXPATHLEN];
	int fd;
	mode_t mask;
	const char *tmpdir, *dot;

	/*
	 * XXX: here we support old naming scheme, such as filename.yaml
	 */
	dot = strrchr(filename, '.');
	if (dot != NULL) {
		snprintf(tmp, MIN(sizeof(tmp), dot - filename + 1), "%s", filename);
		snprintf(url, sizeof(url), "%s/%s.%s", pkg_repo_url(repo), tmp,
				extension);
	}
	else {
		snprintf(url, sizeof(url), "%s/%s.%s", pkg_repo_url(repo), filename,
				extension);
	}

	tmpdir = getenv("TMPDIR");
	if (tmpdir == NULL)
		tmpdir = "/tmp";
	mkdirs(tmpdir);
	snprintf(tmp, sizeof(tmp), "%s/%s.%s.XXXXXX", tmpdir, filename, extension);

	mask = umask(022);
	fd = mkstemp(tmp);
	umask(mask);
	if (fd == -1) {
		pkg_emit_error("Could not create temporary file %s, "
		    "aborting update.\n", tmp);
		*rc = EPKG_FATAL;
		return (-1);
	}
	(void)unlink(tmp);

	if ((*rc = pkg_fetch_file_to_fd(repo, url, fd, t)) != EPKG_OK) {
		close(fd);
		fd = -1;
	}

	return (fd);
}

static bool
pkg_repo_file_has_ext(const char *path, const char *ext)
{
	size_t n, l;
	const char *p = NULL;

	n = strlen(path);
	l = strlen(ext);
	p = &path[n - l];

	if (strcmp(p, ext) == 0)
		return (true);

	return (false);
}

static bool
pkg_repo_check_fingerprint(struct pkg_repo *repo, struct sig_cert *sc, bool fatal)
{
	struct fingerprint *f = NULL;
	char hash[SHA256_DIGEST_LENGTH * 2 + 1];
	int nbgood = 0;
	struct sig_cert *s = NULL, *stmp = NULL;

	if (HASH_COUNT(sc) == 0) {
		if (fatal)
			pkg_emit_error("No signature found");
		return (false);
	}

	/* load fingerprints */
	if (repo->trusted_fp == NULL) {
		if (pkg_repo_load_fingerprints(repo) != EPKG_OK)
			return (false);
	}

	HASH_ITER(hh, sc, s, stmp) {
		if (s->sig == NULL || s->cert == NULL) {
			if (fatal)
				pkg_emit_error("Number of signatures and certificates "
					"mismatch");

			return (false);
		}
		s->trusted = false;
		sha256_buf(s->cert, s->certlen, hash);
		HASH_FIND_STR(repo->revoked_fp, hash, f);
		if (f != NULL) {
			if (fatal)
				pkg_emit_error("At least one of the "
					" certificates has been revoked");

			return (false);
		}

		HASH_FIND_STR(repo->trusted_fp, hash, f);
		if (f != NULL) {
			nbgood++;
			s->trusted = true;
		}
	}

	if (nbgood == 0) {
		if (fatal)
			pkg_emit_error("No trusted public keys found");

		return (false);
	}

	return (true);
}

static void<commit_msg>Get pubkey from meta if possible.<commit_after>/*-
 * Copyright (c) 2011-2013 <NAME> <<EMAIL>>
 * Copyright (c) 2011-2012 <NAME> <<EMAIL>>
 * Copyright (c) 2011-2012 <NAME> <<EMAIL>>
 * Copyright (c) 2012-2013 <NAME> <<EMAIL>>
 * Copyright (c) 2014 <NAME> <<EMAIL>>
 *
 * All rights reserved.
 * 
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer
 *    in this position and unchanged.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include <sys/types.h>
#include <sys/stat.h>
#include <sys/sysctl.h>
#include <sys/uio.h>

#include <archive_entry.h>
#include <assert.h>
#include <fts.h>
#include <libgen.h>
#include <sqlite3.h>
#include <string.h>
#include <dirent.h>
#define _WITH_GETLINE
#include <stdio.h>
#include <stdbool.h>
#include <sysexits.h>
#include <unistd.h>
#include <errno.h>
#include <sys/mman.h>
#include <fcntl.h>

#include "pkg.h"
#include "private/event.h"
#include "private/utils.h"
#include "private/pkg.h"
#include "private/pkgdb.h"
#include "private/repodb.h"
#include "private/thd_repo.h"


struct sig_cert {
	char name[MAXPATHLEN];
	unsigned char *sig;
	int64_t siglen;
	unsigned char *cert;
	int64_t certlen;
	bool cert_allocated;
	UT_hash_handle hh;
	bool trusted;
};

void
pkg_repo_cached_name(struct pkg *pkg, char *dest, size_t destlen)
{
	const char *sum, *name, *version, *reponame, *repourl, *ext = NULL;
	const char *cachedir = NULL;
	struct stat st;

	cachedir = pkg_object_string(pkg_config_get("PKG_CACHEDIR"));

	pkg_get(pkg, PKG_REPONAME, &reponame,
			PKG_CKSUM, &sum, PKG_NAME, &name, PKG_VERSION, &version,
			PKG_REPOPATH, &repourl);

	if (repourl != NULL)
		ext = strrchr(repourl, '.');

	if (ext != NULL) {
		/*
		 * XXX:
		 * This code tries to skip refetching but it should be removed as soon
		 * as we transfer to new scheme.
		 */
		pkg_snprintf(dest, destlen, "%S/%n-%v-%z",
				cachedir, pkg, pkg, pkg);
		if (stat (dest, &st) != -1)
			return;

		/*
		 * The real naming scheme:
		 * <cachedir>/<name>-<version>-<checksum>.txz
		 */
		pkg_snprintf(dest, destlen, "%S/%n-%v-%z%S",
				cachedir, pkg, pkg, pkg, ext);
	}
	else {
		pkg_snprintf(dest, destlen, "%S/%n-%v-%z",
				cachedir, pkg, pkg, pkg);
	}
}

int
pkg_repo_fetch_package(struct pkg *pkg)
{
	char dest[MAXPATHLEN];
	char url[MAXPATHLEN];
	int fetched = 0;
	char cksum[SHA256_DIGEST_LENGTH * 2 +1];
	char *path = NULL;
	const char *packagesite = NULL;

	int retcode = EPKG_OK;
	const char *reponame, *name, *version, *sum;
	struct pkg_repo *repo;

	assert((pkg->type & PKG_REMOTE) == PKG_REMOTE);

	pkg_get(pkg, PKG_REPONAME, &reponame,
			PKG_CKSUM, &sum, PKG_NAME, &name, PKG_VERSION, &version);
	pkg_repo_cached_name(pkg, dest, sizeof(dest));

	/* If it is already in the local cachedir, dont bother to
	 * download it */
	if (access(dest, F_OK) == 0)
		goto checksum;

	/* Create the dirs in cachedir */
	if ((path = dirname(dest)) == NULL) {
		pkg_emit_errno("dirname", dest);
		retcode = EPKG_FATAL;
		goto cleanup;
	}

	if ((retcode = mkdirs(path)) != EPKG_OK)
		goto cleanup;

	/*
	 * In multi-repos the remote URL is stored in pkg[PKG_REPOURL]
	 * For a single attached database the repository URL should be
	 * defined by PACKAGESITE.
	 */
	repo = pkg_repo_find_name(reponame);
	packagesite = pkg_repo_url(repo);

	if (packagesite == NULL || packagesite[0] == '\0') {
		pkg_emit_error("PACKAGESITE is not defined");
		retcode = 1;
		goto cleanup;
	}

	if (packagesite[strlen(packagesite) - 1] == '/')
		pkg_snprintf(url, sizeof(url), "%S%R", packagesite, pkg);
	else
		pkg_snprintf(url, sizeof(url), "%S/%R", packagesite, pkg);

	if (strncasecmp(packagesite, "file://", 7) == 0) {
		pkg_set(pkg, PKG_REPOPATH, url + 7);
		return (EPKG_OK);
	}

	retcode = pkg_fetch_file(repo, url, dest, 0);
	fetched = 1;

	if (retcode != EPKG_OK)
		goto cleanup;

	checksum:
	retcode = sha256_file(dest, cksum);
	if (retcode == EPKG_OK)
		if (strcmp(cksum, sum)) {
			if (fetched == 1) {
				pkg_emit_error("%s-%s failed checksum "
				    "from repository", name, version);
				retcode = EPKG_FATAL;
			} else {
				pkg_emit_error("cached package %s-%s: "
				    "checksum mismatch, fetching from remote",
				    name, version);
				unlink(dest);
				return (pkg_repo_fetch_package(pkg));
			}
		}

	cleanup:
	if (retcode != EPKG_OK)
		unlink(dest);

	return (retcode);
}

static int
pkg_repo_fetch_remote_tmp(struct pkg_repo *repo,
		const char *filename, const char *extension, time_t *t, int *rc)
{
	char url[MAXPATHLEN];
	char tmp[MAXPATHLEN];
	int fd;
	mode_t mask;
	const char *tmpdir, *dot;

	/*
	 * XXX: here we support old naming scheme, such as filename.yaml
	 */
	dot = strrchr(filename, '.');
	if (dot != NULL) {
		snprintf(tmp, MIN(sizeof(tmp), dot - filename + 1), "%s", filename);
		snprintf(url, sizeof(url), "%s/%s.%s", pkg_repo_url(repo), tmp,
				extension);
	}
	else {
		snprintf(url, sizeof(url), "%s/%s.%s", pkg_repo_url(repo), filename,
				extension);
	}

	tmpdir = getenv("TMPDIR");
	if (tmpdir == NULL)
		tmpdir = "/tmp";
	mkdirs(tmpdir);
	snprintf(tmp, sizeof(tmp), "%s/%s.%s.XXXXXX", tmpdir, filename, extension);

	mask = umask(022);
	fd = mkstemp(tmp);
	umask(mask);
	if (fd == -1) {
		pkg_emit_error("Could not create temporary file %s, "
		    "aborting update.\n", tmp);
		*rc = EPKG_FATAL;
		return (-1);
	}
	(void)unlink(tmp);

	if ((*rc = pkg_fetch_file_to_fd(repo, url, fd, t)) != EPKG_OK) {
		close(fd);
		fd = -1;
	}

	return (fd);
}

static bool
pkg_repo_file_has_ext(const char *path, const char *ext)
{
	size_t n, l;
	const char *p = NULL;

	n = strlen(path);
	l = strlen(ext);
	p = &path[n - l];

	if (strcmp(p, ext) == 0)
		return (true);

	return (false);
}

static bool
pkg_repo_check_fingerprint(struct pkg_repo *repo, struct sig_cert *sc, bool fatal)
{
	struct fingerprint *f = NULL;
	char hash[SHA256_DIGEST_LENGTH * 2 + 1];
	int nbgood = 0;
	struct sig_cert *s = NULL, *stmp = NULL;
	struct pkg_repo_meta_key *mk = NULL;

	if (HASH_COUNT(sc) == 0) {
		if (fatal)
			pkg_emit_error("No signature found");
		return (false);
	}

	/* load fingerprints */
	if (repo->trusted_fp == NULL) {
		if (pkg_repo_load_fingerprints(repo) != EPKG_OK)
			return (false);
	}

	HASH_ITER(hh, sc, s, stmp) {
		if (s->sig != NULL && s->cert == NULL) {
			/*
			 * We may want to check meta
			 */
			if (repo->meta != NULL && repo->meta->keys != NULL)
				HASH_FIND_STR(repo->meta->keys, s->name, mk);

			if (mk != NULL && mk->pubkey != NULL) {
				s->cert = mk->pubkey;
				s->certlen = strlen(mk->pubkey);
			}
			else {
				if (fatal)
					pkg_emit_error("No key with name %s has been found", s->name);
				return (false);
			}
		}
		else if (s->sig == NULL) {
			if (fatal)
				pkg_emit_error("No signature with name %s has been found", s->name);
			return (false);
		}

		s->trusted = false;
		sha256_buf(s->cert, s->certlen, hash);
		HASH_FIND_STR(repo->revoked_fp, hash, f);
		if (f != NULL) {
			if (fatal)
				pkg_emit_error("At least one of the "
					" certificates has been revoked");

			return (false);
		}

		HASH_FIND_STR(repo->trusted_fp, hash, f);
		if (f != NULL) {
			nbgood++;
			s->trusted = true;
		}
	}

	if (nbgood == 0) {
		if (fatal)
			pkg_emit_error("No trusted public keys found");

		return (false);
	}

	return (true);
}

static void
{'fraction_non_alphanumeric': 0.1452980946527351, 'fraction_numerical': 0.010202827289489859, 'mean_word_length': 3.8411187146682533, 'pattern_counts': {'":': 0, '<': 77, '<?xml version=': 0, '>': 111, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1021779', 'n_tokens_mistral': 6760, 'n_tokens_neox': 5999, 'n_words': 1746}