<commit_before>#include <err.h>
#include <ctype.h>
#include <assert.h>
#include <errno.h>
#include <libgen.h>
#include <string.h>
#include <unistd.h>

#include "pkg.h"
#include "pkg_event.h"
#include "pkg_private.h"

static int pkg_repos_is_reserved_name(struct pkg_repos *repos, struct pkg_repos_entry *re);
static int pkg_repos_exists_in_pkg(struct pkg *pkg, struct pkg_repos_entry *re);

int
pkg_repo_fetch(struct pkg *pkg)
{
	char dest[MAXPATHLEN];
	char cksum[65];
	char *path;
	char *url;
	int retcode = EPKG_OK;

	assert((pkg->type & PKG_REMOTE) == PKG_REMOTE ||
		(pkg->type & PKG_UPGRADE) == PKG_UPGRADE);

	snprintf(dest, sizeof(dest), "%s/%s", pkg_config("PKG_CACHEDIR"),
			 pkg_get(pkg, PKG_REPOPATH));

	/* If it is already in the local cachedir, dont bother to download it */
	if (access(dest, F_OK) == 0)
		goto checksum;

	/* Create the dirs in cachedir */
	if ((path = dirname(dest)) == NULL) {
		EMIT_ERRNO("dirname", dest);
		retcode = EPKG_FATAL;
		goto cleanup;
	}
	if ((retcode = mkdirs(path)) != 0)
		goto cleanup;

	asprintf(&url, "%s/%s", pkg_config("PACKAGESITE"),
			 pkg_get(pkg, PKG_REPOPATH));

	retcode = pkg_fetch_file(url, dest);
	free(url);
	if (retcode != EPKG_OK)
		goto cleanup;

	checksum:
	retcode = sha256_file(dest, cksum);
	if (retcode == EPKG_OK)
		if (strcmp(cksum, pkg_get(pkg, PKG_CKSUM))) {
			EMIT_FAILED_CKSUM(pkg);
			retcode = EPKG_FATAL;
		}

	cleanup:<commit_msg>Make pkg_fetch_file() aware of the multi repositories<commit_after>#include <err.h>
#include <ctype.h>
#include <assert.h>
#include <errno.h>
#include <libgen.h>
#include <string.h>
#include <unistd.h>

#include "pkg.h"
#include "pkg_event.h"
#include "pkg_private.h"

static int pkg_repos_is_reserved_name(struct pkg_repos *repos, struct pkg_repos_entry *re);
static int pkg_repos_exists_in_pkg(struct pkg *pkg, struct pkg_repos_entry *re);

int
pkg_repo_fetch(struct pkg *pkg)
{
	char dest[MAXPATHLEN];
	char cksum[65];
	char *path = NULL;
	char *url = NULL;
	int retcode = EPKG_OK;
	int fetch_ok = 0;

	struct pkg_repos_entry *re = NULL;

	assert((pkg->type & PKG_REMOTE) == PKG_REMOTE ||
		(pkg->type & PKG_UPGRADE) == PKG_UPGRADE);

	snprintf(dest, sizeof(dest), "%s/%s", pkg_config("PKG_CACHEDIR"),
			 pkg_get(pkg, PKG_REPOPATH));

	/* If it is already in the local cachedir, don't bother to download it */
	if (access(dest, F_OK) == 0)
		goto checksum;

	/* Create the dirs in cachedir */
	if ((path = dirname(dest)) == NULL) {
		EMIT_ERRNO("dirname", dest);
		retcode = EPKG_FATAL;
		goto cleanup;
	}
	if ((retcode = mkdirs(path)) != 0)
		goto cleanup;

	if (pkg_config("PACKAGESITE") != NULL) {
		asprintf(&url, "%s/%s", pkg_config("PACKAGESITE"),
				 pkg_get(pkg, PKG_REPOPATH));

		retcode = pkg_fetch_file(url, dest);
		free(url);
	} else {
		/* 
		 * Get the repository URL from the package itself 
		 * Working on multiple repos here.
		 *
		 * If a package fetch is not successful go to the next
		 * repository in the list and give it more chance for
		 * successful fetch :)
		 */

		re = NULL; /* starts with the first repository entry */
		fetch_ok = EPKG_FATAL;
		while ((pkg_repos_next_in_pkg(pkg, &re) == EPKG_OK) && fetch_ok != EPKG_OK) {
			printf("Fetching package from repository '%s' [%s]\n",
					pkg_repos_get_name(re),
					pkg_repos_get_url(re));

			asprintf(&url, "%s/%s", pkg_repos_get_url(re),
					 pkg_get(pkg, PKG_REPOPATH));

			fetch_ok = retcode = pkg_fetch_file(url, dest);
			free(url);
		}
	}

	if (retcode != EPKG_OK)
		goto cleanup;

	checksum:
	retcode = sha256_file(dest, cksum);
	if (retcode == EPKG_OK)
		if (strcmp(cksum, pkg_get(pkg, PKG_CKSUM))) {
			EMIT_FAILED_CKSUM(pkg);
			retcode = EPKG_FATAL;
		}

	cleanup:
{'fraction_non_alphanumeric': 0.14514378730330982, 'fraction_numerical': 0.004069451980466631, 'mean_word_length': 4.4300441826215025, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7613567', 'n_tokens_mistral': 1636, 'n_tokens_neox': 1455, 'n_words': 352}