<commit_before>/*
 * Copyright (c) 2000-2015 University of Utah and the Flux Group.
 * 
 * {{{EMULAB-LICENSE
 * 
 * This file is part of the Emulab network testbed software.
 * 
 * This file is free software: you can redistribute it and/or modify it
 * under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or (at
 * your option) any later version.
 * 
 * This file is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
 * License for more details.
 * 
 * You should have received a copy of the GNU Affero General Public License
 * along with this file.  If not, see <http://www.gnu.org/licenses/>.
 * 
 * }}}
 */

/*
 * An image zipper.
 *
 * TODO:
 *	Multithread so that we can be reading ahead on the input device
 *	and overlapping IO with compression.  Maybe a third thread for
 *	doing output.
 */
#ifdef WITH_CRYPTO
#include <openssl/evp.h>
#include <openssl/sha.h>
#include <openssl/rsa.h>
#include <openssl/rand.h>
#ifdef SIGN_CHECKSUM
#include <openssl/err.h>
#endif
#endif
#include <ctype.h>
#include <err.h>
#include <fcntl.h>
#if !defined(__UCLIBC__)
#include <fstab.h>
#endif
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <assert.h>
#include <errno.h>
#include <sys/param.h>
#include <sys/time.h>
#include <sys/stat.h>
#include <zlib.h>

#include "imagehdr.h"
#include "sliceinfo.h"
#include "global.h"
#include "checksum.h"
#include "range.h"
#ifdef WITH_HASH
#include "hashmap/hashmap.h"
#endif

/*
 * Attempt to split chunks so that hash blocks don't span chunk boundaries.
 *
 * XXX nice thought, but it doesn't do a very good job (saves less than 50%)
 * of the crossings at the expense of wasting about 4% more space). Unless
 * we come up with a less hacky way to fill chunks (e.g. PD fitblk.c) where
 * we can avoid it entirely, don't even try.
 */
#undef WITH_HASH_CHUNKSPLIT

/* XXX this is a hack right now */
#define USE_HACKSORT 0

#define min(a,b) ((a) <= (b) ? (a) : (b))

char	*infilename;
int	infd, outfd, incanseek, outcanseek;
int	secsize	  = 512;	/* XXX bytes. */
int	debug	  = 0;
int	dots	  = 0;
int	info	  = 0;
int	version	  = 0;
int	slicemode = 0;
int	maxmode	  = 0;
int	slice	  = 0;
int	level	  = 4;
long	dev_bsize = 1;
uint32_t compat   = 0;
int	frangesize= 64;	/* 32k */
int	zerofrange= 0;
int	forcereads= 0;
int	badsectors= 0;
int	retrywrites= 1;
int	dorelocs  = 1;
int	metaoptimize = 0;
int	filemode  = 0;
int	excludenonfs = 0;
int	do_encrypt = 0;
int	cipher = ENC_NONE;
int	do_checksum = 0;
int	csumalg = CSUM_NONE;
off_t	datawritten;
partmap_t ignore, forceraw;
static	int got_imageid;
static unsigned char imageid[UUID_LENGTH];

#ifdef WITH_HASH
char	*hashfile;
char	*newhashfile;
int	deltapct = -1;
#endif

#ifdef WITH_CRYPTO
#ifdef SIGN_CHECKSUM
RSA		*sig_key = NULL;	/* signing key */
static void output_public_key(char *, RSA *);
#endif
unsigned char	*enc_key = NULL;	/* encryption key */
static void output_encrypt_key(char *, unsigned char *, int);
#endif
static void output_uuid(char *, char *);

#define HDRUSED(reg, rel) \
    (sizeof(blockhdr_t) + \
    (reg) * sizeof(struct region) + (rel) * sizeof(struct blockreloc))

/*
 * We want to be able to compress slices by themselves, so we need
 * to know where the slice starts when reading the input file for
 * compression.
 *
 * These numbers are in sectors.
 */
unsigned long inputminsec	= 0;
unsigned long inputmaxsec	= 0;	/* 0 means the entire input image */

struct range	*ranges, *skips, *fixups;
int		numranges, numskips, numfixups;
struct blockreloc	*relocs;
int			numregions, numrelocs;

void	dumpskips(int verbose);
static void	sortrange(struct range **head, int domerge,
		  int (*rangecmp)(struct range *, struct range *));
int	mergeskips(int verbose);
int	mergeranges(struct range *head);
void	makeranges(void);
void	dumpranges(int verbose);
uint32_t sectinranges(struct range *range);
void	addvalid(uint32_t start, uint32_t size);
void	addreloc(off_t offset, off_t size, int reloctype);
void	removereloc(off_t offset, off_t size, int reloctype);
static int cmpfixups(struct range *r1, struct range *r2);

/* Forward decls */
int	read_image(int fd);
int	read_raw(int fd);
int	compress_image(void);
void	usage(void);

static SLICEMAP_PROCESS_PROTO(read_slice);

#define READ_RETRIES	1
#define WRITE_RETRIES	10

ssize_t
slowread(int fd, void *buf, size_t nbytes, off_t startoffset)
{
	int cc, i, count;

	assert(fd != infd || incanseek);

	fprintf(stderr, "read failed: will retry by sector %d more times\n",
		READ_RETRIES);

	count = 0;
	for (i = 0; i < READ_RETRIES; i++) {
		if (lseek(fd, startoffset, SEEK_SET) < 0) {
			perror("devread: seeking to set file ptr");
			exit(1);
		}
		while (nbytes > 0) {
			cc = read(fd, buf, nbytes);
			if (cc == 0) {
				nbytes = 0;
				continue;
			}
			if (cc > 0) {
				nbytes -= cc;
				buf     = (void *)((char *)buf + cc);
				count  += cc;
				continue;
			}

			nbytes += count;
			buf     = (void *)((char *)buf - count);
			count   = 0;
			break;
		}
		if (nbytes == 0)
			break;
	}
	return count;
}

/*
 * Assert the hell out of it...
 */
off_t
devlseek(int fd, off_t off, int whence)
{
	off_t noff;

	assert(fd != infd || incanseek);
	assert((off & (DEV_BSIZE-1)) == 0);
	noff = lseek(fd, off, whence);
	if (!filemode) {
		assert(noff == (off_t)-1 || (noff & (DEV_BSIZE-1)) == 0);
	}
	return noff;
}

/*
 * Wrap up read in a retry mechanism to persist in the face of IO errors,
 * even faking data if requested.
 */
ssize_t
devread(int fd, void *buf, size_t nbytes)
{
	int		cc, count;
	off_t		startoffset;
	size_t		onbytes;

#ifndef linux
	assert((nbytes & (DEV_BSIZE-1)) == 0);
#endif
	cc = read(fd, buf, nbytes);

	/*
	 * If reading from a pipe, try to fill the input buffer
	 * even if it takes multiple reads. Forces deterministic
	 * behavior when compressing the same input stream.
	 */
	if (cc > 0 && cc != nbytes && !incanseek) {
		int ncc;

		count = nbytes - cc;
		while (count > 0) {
			ncc = read(fd, (char *)buf + cc, count);
			if (ncc <= 0)
				break;
			cc += ncc;
			count -= ncc;
		}
		return cc;
	}

	if (!forcereads || cc >= 0)
		return cc;

	assert(fd != infd || incanseek);

	/*
	 * Got an error reading the range.
	 * Retry one sector at a time, replacing any bad sectors with
	 * zeroed data.
	 */
	if ((startoffset = lseek(fd, (off_t) 0, SEEK_CUR)) < 0) {
		perror("devread: seeking to get input file ptr");
		exit(1);
	}
	assert((startoffset & (DEV_BSIZE-1)) == 0);

	onbytes = nbytes;
	while (nbytes > 0) {
		if (nbytes > DEV_BSIZE)
			count = DEV_BSIZE;
		else
			count = nbytes;
		cc = slowread(fd, buf, count, startoffset);
		if (cc != count) {
			fprintf(stderr, "devread: read failed on sector %u, "
				"returning zeros\n",
				bytestosec(startoffset));
			if (cc < 0)
				cc = 0;
			memset(buf, 0, count-cc);
			badsectors++;
			cc = count;
		}
		nbytes -= cc;
		buf = (void *)((char *)buf + cc);
		startoffset += cc;
	}
	return onbytes;
}

/*
 * Wrap up write in a retry mechanism to protect against transient NFS
 * errors causing a fatal error.
 */
ssize_t
devwrite(int fd, const void *buf, size_t nbytes)
{
	int		cc, i, count = 0;
	off_t		startoffset = 0;

	if (retrywrites && outcanseek &&
	    ((startoffset = lseek(fd, (off_t) 0, SEEK_CUR)) < 0)) {
		perror("devwrite: seeking to get output file ptr");
		exit(1);
	}

	for (i = 0; i < WRITE_RETRIES; i++) {
		while (nbytes) {
			cc = write(fd, buf, nbytes);

			if (cc > 0) {
				nbytes -= cc;
				buf     = (void *)((char *)buf + cc);
				count  += cc;
				continue;
			}

			if (!retrywrites)
				return cc;

			if (i == 0)
				perror("write error: will retry");

			sleep(1);
			nbytes += count;
			buf     = (void *)((char *)buf - count);
			count   = 0;
			goto again;
		}
		if (retrywrites && fsync(fd) < 0) {
			perror("fsync error: will retry");
			sleep(1);
			nbytes += count;
			buf     = (void *)((char *)buf - count);
			count   = 0;
			goto again;
		}
		datawritten += count;
		return count;
	again:
		if (lseek(fd, startoffset, SEEK_SET) < 0) {
			perror("devwrite: seeking to set file ptr");
			exit(1);
		}
	}
	perror("write error: busted for too long");
	fflush(stderr);
	exit(1);
}

static int
setpartition(partmap_t map, char *str)
{
	int dospart;
	char bsdpart;

	if (isdigit(str[1])) {
		bsdpart = str[2];
		str[2] = '\0';
	} else {
		bsdpart = str[1];
		str[1] = '\0';
	}
	dospart = atoi(str);
	if (dospart < 1 || dospart > MAXSLICES)
		return EINVAL;

	/* common case: apply to complete DOS partition */
	if (bsdpart == '\0') {
		map[dospart-1] = ~0;
		return 0;
	}

	if (bsdpart < 'a' || bsdpart > 'p')
		return EINVAL;

	map[dospart-1] |= (1 << (bsdpart - 'a'));
	return 0;
}

int
main(int argc, char *argv[])
{
	int	ch, rval;
	char	*outfilename = 0;
	int	rawmode	  = 0;
	int	slicetype = 0;
	struct timeval sstamp;
	extern char build_info[];

	memset(imageid, '\0', UUID_LENGTH);

	gettimeofday(&sstamp, 0);
	while ((ch = getopt(argc, argv, "vlbnNdihrs:c:z:ofI:13F:DR:S:XxH:U:P:Me:k:u:a:Z")) != -1)
		switch(ch) {
		case 'v':
			version++;
			break;
		case 'i':
			info++;
			debug++;
			break;
		case 'D':
			retrywrites = 0;
			break;
		case 'd':
			debug++;
			break;
		case 'l':
			slicetype = IZTYPE_LINUX;
			break;
		case 'b':
			slicetype = IZTYPE_FBSDNOLABEL;
			break;
		case 'N':
			dorelocs = 0;
			break;
		case 'n':
			slicetype = IZTYPE_NTFS;
			break;
		case 'o':
			dots++;
			break;
		case 'r':
			rawmode++;
			break;
		case 'S':
			slicetype = atoi(optarg);
			/* XXX special case for freebsd */
			if (slicetype == IZTYPE_386BSD)
				slicetype = IZTYPE_FBSDNOLABEL;
			break;
		case 's':
			slicemode = 1;
			slice = atoi(optarg);
			break;
		case 'z':
			level = atoi(optarg);
			if (level < 0 || level > 9)
				usage();
			break;
		case 'c':
			maxmode     = 1;
			inputmaxsec = atoi(optarg);
			break;
		case 'I':
			if (setpartition(ignore, optarg))
				usage();
			break;
		case 'R':
			if (setpartition(forceraw, optarg))
				usage();
			break;
		case '1':
			compat = COMPRESSED_V1;
			break;
		case '3':
			compat = COMPRESSED_V3;
			break;
		case 'F':
			frangesize = atoi(optarg);
			if (frangesize < 0)
				usage();
			break;
		case 'Z':
			zerofrange = 1;
			break;
		case 'x':
			excludenonfs++;
			break;
		case 'X':
			forcereads++;
			break;
#ifdef WITH_HASH
		case 'H':
			hashfile = optarg;
			break;
		case 'U':
			newhashfile = optarg;
			break;
		case 'P':
			deltapct = atoi(optarg);
			if (deltapct < 0)
				usage();
			break;
#else
		case 'H':
		case 'U':
		case 'P':
			fprintf(stderr, "'%c' option not supported\n", ch);
			usage();
			break;
#endif
		case 'M':
			metaoptimize++;
			break;
		case 'f':
			filemode++;
			rawmode++;
			break;
		case 'a':
#ifdef WITH_CRYPTO
			/* Authentication (checksum) hash algorithm */
			if (strcmp(optarg, "SHA1") == 0) {
				csumalg = CSUM_SHA1;
			}
			else {
				fprintf(stderr, "Only know \"SHA1\"\n");
				usage();
			}
			do_checksum++;
#ifdef SIGN_CHECKSUM
			csumalg |= CSUM_SIGNED;
#else
			fprintf(stderr, "WARNING: checksum is not signed\n");
#endif
#else
			fprintf(stderr, "'a' option not supported\n");
			usage();
#endif
			break;

		case 'e':
#ifdef WITH_CRYPTO
			/* Encryption cipher */
			if (strcmp(optarg, "bf_cbc") == 0) {
				cipher = ENC_BLOWFISH_CBC;
			}
			else {
				fprintf(stderr,
					"Only know \"bf_cbc\" (blowfish CBC)\n");
				usage();
			}
			do_encrypt++;
#else
			fprintf(stderr, "'e' option not supported\n");
			usage();
#endif
			break;
		case 'k':
#ifdef WITH_CRYPTO
			/* Encryption key file */
			enc_key = calloc(1, ENC_MAX_KEYLEN);
			if (enc_key == NULL ||
			    !encrypt_readkey(optarg, enc_key, ENC_MAX_KEYLEN))
				usage();
			/* XXX can you intuit the cipher from the key? */
			if (cipher == ENC_NONE)
				cipher = ENC_BLOWFISH_CBC;
#else
			fprintf(stderr, "'k' option not supported\n");
			usage();
#endif
			break;
		case 'u':
			/* UUID for image id. */
			if (!hexstr_to_mem(imageid, optarg, UUID_LENGTH))
				usage();
			got_imageid = 1;
			break;
		case 'h':
		case '?':
		default:
			usage();
		}
	argc -= optind;
	argv += optind;

	if (version || debug) {
		fprintf(stderr, "%s\n", build_info);
		if (version) {
			char *str;

			fprintf(stderr, "Supports:\n");
			fprintf(stderr, "  Partitioning: ");
#ifdef WITH_MBR
			fprintf(stderr, "MBR");
			str = ",";
#endif
#ifdef WITH_GPT
			fprintf(stderr, "%sGPT", str);
#endif
			fprintf(stderr, "\n  Partition types: ");
			printslicemap();
			fprintf(stderr, "\n  Features: ");
			fprintf(stderr, "image UUIDs");
#ifdef WITH_CRYPTO
#ifdef SIGN_CHECKSUM
			fprintf(stderr, ",signed");
#else
			fprintf(stderr, ",unsigned");
#endif
			fprintf(stderr, " SHA1 checksums");
			fprintf(stderr, ",blowfish encryption");
#endif
#ifdef WITH_HASH
			fprintf(stderr, ",delta image creation");
#endif
			fprintf(stderr, "\n");
			exit(0);
		}
	}

	if (argc < 1 || argc > 2)
		usage();

	if (compat &&
	    (
#ifdef WITH_CRYPTO
	     do_encrypt || do_checksum ||
#endif
	     got_imageid)) {
		fprintf(stderr, "Cannot use uuid/encrypt/checksum with -3\n");
		usage();
	}
	if (slicemode && (slice < 1 || slice > MAXSLICES)) {
		fprintf(stderr, "Slice must be a DOS partition (1-4) "
			"or extended DOS partition (5-%d)\n\n", MAXSLICES);
		usage();
	}
	if (maxmode && slicemode) {
		fprintf(stderr, "Count option (-c) cannot be used with "
			"the slice (-s) option\n\n");
		usage();
	}
	if (!info && argc != 2) {
		fprintf(stderr, "Must specify an output filename!\n\n");
		usage();
	}
	else
		outfilename = argv[1];

	if (!slicemode && !filemode && dorelocs)
		dorelocs = 0;

	infilename = argv[0];
	if (strcmp(infilename, "-")) {
		if ((infd = open(infilename, O_RDONLY, 0)) < 0) {
			perror(infilename);
			exit(1);
		}
		incanseek = 1;
	}
	else {
		if (!rawmode) {
			fprintf(stderr,
				"Can only use stdin as input with -f or -r\n");
			usage();
		}
		infd = fileno(stdin);
		incanseek = 0;
		forcereads = 0;
	}

#ifdef WITH_CRYPTO
	/*
	 * Let's get random!
	 */
	if (!RAND_load_file("/dev/urandom", 1024))
		fprintf(stderr, "Error getting random seed\n");

#ifdef SIGN_CHECKSUM
	/*
	 * Generate a signing pubkey.
	 * The pubkey is written to <imagename>.skey.
	 */
	if (do_checksum) {
		sig_key = RSA_generate_key(CSUM_MAX_LEN*8, 17, NULL, NULL);
		if (!info)
			output_public_key(outfilename, sig_key);
	}
#endif

	/*
	 * Generate an encryption key if none was given on the command line.
	 * The key is written to <imagename>.ekey.
	 */
	if (do_encrypt && enc_key == NULL) {
		enc_key = calloc(1, ENC_MAX_KEYLEN);
		if (enc_key == NULL || !RAND_bytes(enc_key, ENC_MAX_KEYLEN)) {
			fprintf(stderr, "Unable to generate random key\n");
			exit(1);
		}
		if (!info)
			output_encrypt_key(outfilename, enc_key,
					   ENC_MAX_KEYLEN);
	}
#endif

#ifdef WITH_V3COMPAT
	/*
	 * Deal with pre-crypto backward compatibility.
	 * We don't want to generate V4 images unless we really need it.
	 * Note that this means we do NOT generate a UUID if none was provided.
	 */
	if (!compat &&
#ifdef WITH_CRYPTO
	    !do_encrypt && !do_checksum &&
#endif
	    !got_imageid)
		compat = COMPRESSED_V3;
#endif

	/*
	 * Generate a random UUID if one was not provided and we are
	 * not operating in compatibility mode.
	 */
	if (!compat && !got_imageid) {
		int fd = open("/dev/urandom", O_RDONLY, 0);

		if (fd < 0 ||
		    read(fd, imageid, sizeof(imageid)) != sizeof(imageid)) {
			fprintf(stderr, "WARNING: no UUID generated\n");
			memset(imageid, '\0', UUID_LENGTH);
		} else {
			char uuidstr[UUID_LENGTH*2+1];

			mem_to_hexstr(uuidstr, imageid, UUID_LENGTH);
			output_uuid(outfilename, uuidstr);
			got_imageid = 1;
		}

		if (fd >= 0)
			close(fd);
	}

	/*
	 * Create the skip list by scanning the filesystems on
	 * the disk or indicated partition.
	 */
	if (slicetype != 0) {
		/*
		 * See if we can get a meaningful max size value via seek
		 */
		if (inputmaxsec == 0) {
			off_t maxoff = lseek(infd, (off_t)0, SEEK_END);
			if (maxoff != (off_t)-1 && maxoff != (off_t)0) {
				inputmaxsec = bytestosec(maxoff);
				assert(inputmaxsec == 0 ||
				       inputmaxsec > inputminsec);
				if (debug)
					fprintf(stderr, "Max sector: %lu\n",
						inputmaxsec);
			}
			(void) lseek(infd, (off_t)0, SEEK_SET);
		}

		rval = read_slice(-1, slicetype, 0, inputmaxsec,
				  infilename, infd);
		if (rval == -1)
			fprintf(stderr, ", cannot process\n");
	} else if (rawmode)
		rval = read_raw(infd);
	else
		rval = read_image(infd);
	if (rval) {
		fprintf(stderr, "* * * Aborting * * *\n");
		exit(1);
	}

	/*
	 * Create a valid range list from the skip list
	 */
	(void) mergeskips(debug > 1);
	if (debug)
		dumpskips(debug > 1);
	makeranges();
	if (debug)
		dumpranges(debug > 1);
	if (ranges == NULL) {
		/*
		 * No valid ranges, exit with an error.
		 *
		 * XXX it is debatable whether this is really an error.
		 * Perhaps we should just create a valid image file with
		 * one chunk containing no valid ranges, but it seems likely
		 * that they probably made a mistake and we should call it
		 * out immediately.
		 */
		fprintf(stderr,
			"*** No valid data on specified disk/partition!?\n");
		exit(1);
	}
	sortrange(&fixups, 0, cmpfixups);
	if (debug > 1)
		dumpfixups(debug > 2, 0);
	fflush(stderr);

#ifdef WITH_HASH
	/*
	 * If we are creating a "delta" image from a hash signature,
	 * (hashfile != NULL) we read in the signature info and reconcile
	 * that with the known allocated range that we have just computed.
	 * The result is a new list of ranges that are currently allocated
	 * and that have changed from the signature version.
	 *
	 * If we are creating a new signature file (newhashfile != NULL)
	 * then we also collect hashinfo along the way, writing out the
	 * newfile when done.
	 */
	if (hashfile || newhashfile) {
		struct range *nranges = NULL;

		/*
		 * next compare allocated 'ranges' and 'hinfo' to find out the
		 * changed blocks -- computing the hashes for some 'ranges'
		 * in the process
		 */
		if (hashmap_compute_delta(ranges, hashfile, infd, inputminsec,
					  newhashfile, &nranges)) {
			fprintf(stderr, "Errors while computing delta!\n");
			exit(1);
		}

		/*
		 * See if we should just create a full image.
		 */
		if (hashfile && deltapct >= 0) {
			uint32_t oldsect = sectinranges(ranges);
			uint32_t newsect = sectinranges(nranges);
			int dofull =
				(oldsect == 0 ||
				 ((double)newsect / oldsect) * 100 > deltapct);
			fprintf(stderr,
				"Full image size %u sect, "
				"delta image size %u sect\n"
				"Auto image selection creating %s image.\n",
				oldsect, newsect, dofull ? "full" : "delta");
			if (dofull) {
				freeranges(nranges);
				goto done;
			}
		}

		freeranges(ranges);
		ranges = nranges;
		if (debug) {
			fprintf(stderr, "\nAfter delta computation: ");
			dumpranges(debug > 1);
			dumpfixups(debug > 2, 0);
		}
	done:
		hashmap_dump_stats(slice);
	}
#endif

	/*
	 * Now we have all the allocated information, create the image
	 * (unless we just want an info report).
	 */
	if (!info) {
		if (strcmp(outfilename, "-")) {
			if ((outfd = open(outfilename, O_RDWR|O_CREAT|O_TRUNC,
					  0666)) < 0) {
				perror("opening output file");
				exit(1);
			}
			outcanseek = 1;
			/*
			 * XXX retrywrites is a hack to avoid transient NFS
			 * write errors: we use fsync after every write to
			 * check that it worked, re-trying if not. However,
			 * some devices (e.g., /dev/null) do not support
			 * fsync, so check for that here and disable the
			 * retry if necessary.
			 */
			if (retrywrites &&
			    fsync(outfd) < 0 && errno == EINVAL) {
				fprintf(stderr,
					"WARNING: '%s' does not support fsync,"
					" write errors may not be detected "
					"or corrected.\n", outfilename);
				retrywrites = 0;
			}
		}
		else {
			outfd = fileno(stdout);
			outcanseek = 0;
			retrywrites = 0;
		}
		compress_image();
		assert(fixups == NULL);

		if (outcanseek)
			close(outfd);

#ifdef WITH_HASH
		/*
		 * Write out new signature file.
		 */
		if (newhashfile &&
		    hashmap_write_hashfile(newhashfile, outfilename))
			fprintf(stderr, "Could not write new hashfile!\n");
#endif
	}
	close(infd);

	{
		struct timeval stamp;
		unsigned int ms;

		gettimeofday(&stamp, 0);
		if (stamp.tv_usec < sstamp.tv_usec) {
			stamp.tv_usec += 1000000;
			stamp.tv_sec--;
		}
		ms = (stamp.tv_sec - sstamp.tv_sec) * 1000 +
			(stamp.tv_usec - sstamp.tv_usec) / 1000;
		fprintf(stderr,
			"Finished in %u.%03u seconds\n",
			ms / 1000, ms % 1000);
	}
	fflush(stderr);

#ifdef WITH_CRYPTO
#ifdef SIGN_CHECKSUM
	if (do_checksum)
		RSA_free(sig_key);
#endif
#endif

	exit(0);
}

static int
read_slice(int snum, iz_type stype, iz_lba start, iz_size size,
	   char *sname, int sfd)
{
	struct sliceinfo *smap = getslicemap(stype);

	if (smap && smap->process)
		return (*smap->process)(snum, stype, start, size, sname, sfd);

	fprintf(stderr, "Slice %d is an unknown type %#x (%s)",
		snum+1, stype, smap ? smap->desc : "??");
	return -1;
}

/*
 * Parse the MBR/GPT and dispatch to the individual readers.
 */
int
read_image(int fd)
{
	int		i, rval = 0;
	struct iz_disk	disk;
	struct iz_slice	*parttab;
	int		gotbb = 0;
	char *		bbstr;
	iz_lba		dstart;
	iz_size		dsize;
	int		dowarn = debug ? 2 : 0;

#ifdef WITH_GPT
	if (!gotbb && parse_gpt(fd, &disk, dowarn) == 0) {
		gotbb = 1;
		bbstr = "GPT";
	}
#endif
#ifdef WITH_MBR
	if (!gotbb && parse_mbr(fd, &disk, dowarn) == 0) {
		gotbb = 2;
		bbstr = "MBR";
	}
#endif

	if (!gotbb) {
		/* try to parse as one of the FS types */
		warnx("No GPT or MBR partition table found!");
		return 1;
	}

	parttab = disk.slices;
	dstart = 0;
	dsize = disk.dsize;

	if (debug) {
		int i;

		fprintf(stderr, "Disk:            start %12d, size %12d\n",
			0, dsize);
		fprintf(stderr, "Usable:          start %12d, size %12d\n",
			disk.lodata, disk.hidata - disk.lodata + 1);
		fprintf(stderr, "Partition range: start %12d, size %12d\n",
			disk.losect, disk.hisect - disk.losect + 1);
		fprintf(stderr, "%s Partitions:\n", bbstr);
		for (i = 0; i < MAXSLICES; i++) {
			struct sliceinfo *sinfo;

			if (parttab[i].type == IZTYPE_INVALID)
				continue;

			fprintf(stderr, "  P%d: ", i+1);
			sinfo = getslicemap(parttab[i].type);
			if (sinfo == 0)
				fprintf(stderr, "0x%x", parttab[i].type);
			else
				fprintf(stderr, "%-12s", sinfo->desc);

			fprintf(stderr, "  start %12d, size %12d",
				parttab[i].offset, parttab[i].size);
			if (parttab[i].flags) {
				fprintf(stderr, " (");
				if (parttab[i].flags & IZFLAG_NOTSUP)
					fprintf(stderr, "Not supported,");
				if (parttab[i].flags & IZFLAG_IGNORE)
					fprintf(stderr, "IGNORED,");
				if (parttab[i].flags & IZFLAG_RAW)
					fprintf(stderr, "compress RAW");
				fprintf(stderr, ")\n");
			} else {
				fprintf(stderr, "\n");
			}
		}
		fprintf(stderr, "\n");
	}

	/*
	 * Quick, brute-force check for overlap of partitions.
	 * XXX right now, any overlap is bad and we bail.  In the future,
	 * we could determine all areas of intersection and be conservative
	 * with those areas; i.e., always save unless overlap is strictly
	 * between unused/ignored partitions.
	 */
	for (i = 0; i < MAXSLICES; i++) {
		iz_lba start1, start2;
		iz_size size1, size2;
		int ii;

		if (parttab[i].type == IZTYPE_INVALID)
			continue;

		if ((size1 = parttab[i].size) == 0)
			continue;

		start1 = parttab[i].offset;
		for (ii = i + 1; ii < MAXSLICES; ii++) {
			if (parttab[ii].type == IZTYPE_INVALID)
				continue;
			if ((size2 = parttab[ii].size) == 0)
				continue;
			start2 = parttab[ii].offset;
			if (start2+size2 > start1 &&
			    start1+size1 > start2) {
				warnx("P%d and P%d overlap!", i+1, ii+1);
				rval++;
			}
		}
	}
	if (rval)
		return 1;

	/*
	 * If not in slice mode, skip sectors outside of any partition.
	 *
	 * XXX we could skip space between partitions too since we don't
	 * allow overlapping partitions. But partitions don't have to be
	 * in lowest to highest order on the disk, so it is harder.
	 */
	if (!slicemode) {
		iz_lba dlow = disk.lodata;
		iz_lba dhigh = disk.hidata;
		iz_lba losect = disk.losect;
		iz_lba hisect = disk.hisect;

		if (losect > dlow) {
			addskip(dlow, losect-dlow);
			if (dowarn > 1)
				warnx("%s: skipping %u sectors at %u",
				      bbstr, losect - dlow, dlow);
		}
		if (hisect < dhigh) {
			addskip(hisect+1, dhigh-hisect);
			if (dowarn > 1)
				warnx("%s: skipping %lu sectors at %u",
				      bbstr, (unsigned long)(dhigh - hisect),
				      hisect + 1);
		}
	}

	/*
	 * Now operate on individual slices.
	 */
	for (i = 0; i < MAXSLICES; i++) {
		iz_type	type  = parttab[i].type;
		iz_lba start  = parttab[i].offset;
		iz_size size  = parttab[i].size;
		iz_flags flags= parttab[i].flags;

		/*
		 * No such slice
		 */
		if (type == IZTYPE_INVALID)
			continue;

		/*
		 * This is not the slice you are looking for...
		 */
		if (slicemode && i + 1 != slice)
			continue;

		/*
		 * Explicit request by the user to ignore the partition
		 */
		if (ignore[i] && (!ISBSD(type) || ignore[i] == ~0)) {
			fprintf(stderr,
				"P%d: forcing skip, NOT SAVING.\n", i+1);
			flags |= IZFLAG_IGNORE;
		}
		/*
		 * Explicit request by the user to compress raw
		 */
		else if (forceraw[i] && (!ISBSD(type) || forceraw[i] == ~0)) {
			fprintf(stderr,
				"P%d: forcing raw compression.\n", i+1);
			flags |= IZFLAG_RAW;
		}
		/*
		 * Partition type is unsupported. We could either compress
		 * it raw or ignore it. Right now we always compress them
		 * raw. We could add an option to ignore them.
		 */
		else if ((flags & IZFLAG_NOTSUP) != 0) {
			fprintf(stderr,
				"P%d: Type 0x%x not supported, ", i+1, type);

			fprintf(stderr, "forcing raw compression.\n");
			flags |= IZFLAG_RAW;
		}
		/*
		 * Partition is being ignored either because it is unused
		 * or otherwise contains no useful info (e.g., extended DOS
		 * partitions).
		 */
		else if ((flags & IZFLAG_IGNORE) != 0) {
			fprintf(stderr,
				"P%d: %s, NOT SAVING.\n", i+1,
				type == IZTYPE_UNUSED ? "unused" : "ignored");
			if (size > 0)
				addskip(start, size);
		}
		/*
		 * Finally! We have a partition that we can apply some
		 * smarts to. If that fails, we try raw compression.
		 */
		else {
			rval = read_slice(i, type, start, size,
					  infilename, fd);
			if (rval == -1) {
				fprintf(stderr, ", forcing raw compression\n");
				rval = 0;
			}
		}

		/*
		 * In slicemode, we need to set the bounds of compression.
		 * If not in slicemode, we cannot set the bounds according
		 * to the partition info since its possible that someone
		 * will create a disk with empty space before the first
		 * partition (typical, to start partition 1 at the second
		 * cylinder) or after the last partition (Mike!). However,
		 * do not set the inputminsec since we usually want the
		 * stuff before the first partition, which is the boot stuff.
		 */
		if (slicemode) {
			inputminsec = start;
			inputmaxsec = start + size;
		} else if (!maxmode && (start + size) > inputmaxsec) {
			inputmaxsec = start + size;
		}
	}

	/*
	 * Ultimately, we are going to believe the partitioner about
	 * min/max values, but lets see how they compare.
	 */
	if (!slicemode) {
		if (dstart != inputminsec) {
			fprintf(stderr,
				"partitioner value (%lu) different than "
				"computed value (%lu); using the former\n",
				(unsigned long)dstart, inputminsec);
			inputminsec = dstart;
		}
		if (!maxmode && dstart+dsize != inputmaxsec) {
			fprintf(stderr,
				"partitioner value (%lu) different than "<commit_msg>Ignore "historical bogus MBR" and bug fix for maxmode (-c).<commit_after>/*
 * Copyright (c) 2000-2015 University of Utah and the Flux Group.
 * 
 * {{{EMULAB-LICENSE
 * 
 * This file is part of the Emulab network testbed software.
 * 
 * This file is free software: you can redistribute it and/or modify it
 * under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or (at
 * your option) any later version.
 * 
 * This file is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
 * License for more details.
 * 
 * You should have received a copy of the GNU Affero General Public License
 * along with this file.  If not, see <http://www.gnu.org/licenses/>.
 * 
 * }}}
 */

/*
 * An image zipper.
 *
 * TODO:
 *	Multithread so that we can be reading ahead on the input device
 *	and overlapping IO with compression.  Maybe a third thread for
 *	doing output.
 */
#ifdef WITH_CRYPTO
#include <openssl/evp.h>
#include <openssl/sha.h>
#include <openssl/rsa.h>
#include <openssl/rand.h>
#ifdef SIGN_CHECKSUM
#include <openssl/err.h>
#endif
#endif
#include <ctype.h>
#include <err.h>
#include <fcntl.h>
#if !defined(__UCLIBC__)
#include <fstab.h>
#endif
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <assert.h>
#include <errno.h>
#include <sys/param.h>
#include <sys/time.h>
#include <sys/stat.h>
#include <zlib.h>

#include "imagehdr.h"
#include "sliceinfo.h"
#include "global.h"
#include "checksum.h"
#include "range.h"
#ifdef WITH_HASH
#include "hashmap/hashmap.h"
#endif

/*
 * Attempt to split chunks so that hash blocks don't span chunk boundaries.
 *
 * XXX nice thought, but it doesn't do a very good job (saves less than 50%)
 * of the crossings at the expense of wasting about 4% more space). Unless
 * we come up with a less hacky way to fill chunks (e.g. PD fitblk.c) where
 * we can avoid it entirely, don't even try.
 */
#undef WITH_HASH_CHUNKSPLIT

/* XXX this is a hack right now */
#define USE_HACKSORT 0

#define min(a,b) ((a) <= (b) ? (a) : (b))

char	*infilename;
int	infd, outfd, incanseek, outcanseek;
int	secsize	  = 512;	/* XXX bytes. */
int	debug	  = 0;
int	dots	  = 0;
int	info	  = 0;
int	version	  = 0;
int	slicemode = 0;
int	maxmode	  = 0;
int	slice	  = 0;
int	level	  = 4;
long	dev_bsize = 1;
uint32_t compat   = 0;
int	frangesize= 64;	/* 32k */
int	zerofrange= 0;
int	forcereads= 0;
int	badsectors= 0;
int	retrywrites= 1;
int	dorelocs  = 1;
int	metaoptimize = 0;
int	filemode  = 0;
int	excludenonfs = 0;
int	do_encrypt = 0;
int	cipher = ENC_NONE;
int	do_checksum = 0;
int	csumalg = CSUM_NONE;
off_t	datawritten;
partmap_t ignore, forceraw;
static	int got_imageid;
static unsigned char imageid[UUID_LENGTH];

#ifdef WITH_HASH
char	*hashfile;
char	*newhashfile;
int	deltapct = -1;
#endif

#ifdef WITH_CRYPTO
#ifdef SIGN_CHECKSUM
RSA		*sig_key = NULL;	/* signing key */
static void output_public_key(char *, RSA *);
#endif
unsigned char	*enc_key = NULL;	/* encryption key */
static void output_encrypt_key(char *, unsigned char *, int);
#endif
static void output_uuid(char *, char *);

#define HDRUSED(reg, rel) \
    (sizeof(blockhdr_t) + \
    (reg) * sizeof(struct region) + (rel) * sizeof(struct blockreloc))

/*
 * We want to be able to compress slices by themselves, so we need
 * to know where the slice starts when reading the input file for
 * compression.
 *
 * These numbers are in sectors.
 */
unsigned long inputminsec	= 0;
unsigned long inputmaxsec	= 0;	/* 0 means the entire input image */

struct range	*ranges, *skips, *fixups;
int		numranges, numskips, numfixups;
struct blockreloc	*relocs;
int			numregions, numrelocs;

void	dumpskips(int verbose);
static void	sortrange(struct range **head, int domerge,
		  int (*rangecmp)(struct range *, struct range *));
int	mergeskips(int verbose);
int	mergeranges(struct range *head);
void	makeranges(void);
void	dumpranges(int verbose);
uint32_t sectinranges(struct range *range);
void	addvalid(uint32_t start, uint32_t size);
void	addreloc(off_t offset, off_t size, int reloctype);
void	removereloc(off_t offset, off_t size, int reloctype);
static int cmpfixups(struct range *r1, struct range *r2);

/* Forward decls */
int	read_image(int fd);
int	read_raw(int fd);
int	compress_image(void);
void	usage(void);

static SLICEMAP_PROCESS_PROTO(read_slice);

#define READ_RETRIES	1
#define WRITE_RETRIES	10

ssize_t
slowread(int fd, void *buf, size_t nbytes, off_t startoffset)
{
	int cc, i, count;

	assert(fd != infd || incanseek);

	fprintf(stderr, "read failed: will retry by sector %d more times\n",
		READ_RETRIES);

	count = 0;
	for (i = 0; i < READ_RETRIES; i++) {
		if (lseek(fd, startoffset, SEEK_SET) < 0) {
			perror("devread: seeking to set file ptr");
			exit(1);
		}
		while (nbytes > 0) {
			cc = read(fd, buf, nbytes);
			if (cc == 0) {
				nbytes = 0;
				continue;
			}
			if (cc > 0) {
				nbytes -= cc;
				buf     = (void *)((char *)buf + cc);
				count  += cc;
				continue;
			}

			nbytes += count;
			buf     = (void *)((char *)buf - count);
			count   = 0;
			break;
		}
		if (nbytes == 0)
			break;
	}
	return count;
}

/*
 * Assert the hell out of it...
 */
off_t
devlseek(int fd, off_t off, int whence)
{
	off_t noff;

	assert(fd != infd || incanseek);
	assert((off & (DEV_BSIZE-1)) == 0);
	noff = lseek(fd, off, whence);
	if (!filemode) {
		assert(noff == (off_t)-1 || (noff & (DEV_BSIZE-1)) == 0);
	}
	return noff;
}

/*
 * Wrap up read in a retry mechanism to persist in the face of IO errors,
 * even faking data if requested.
 */
ssize_t
devread(int fd, void *buf, size_t nbytes)
{
	int		cc, count;
	off_t		startoffset;
	size_t		onbytes;

#ifndef linux
	assert((nbytes & (DEV_BSIZE-1)) == 0);
#endif
	cc = read(fd, buf, nbytes);

	/*
	 * If reading from a pipe, try to fill the input buffer
	 * even if it takes multiple reads. Forces deterministic
	 * behavior when compressing the same input stream.
	 */
	if (cc > 0 && cc != nbytes && !incanseek) {
		int ncc;

		count = nbytes - cc;
		while (count > 0) {
			ncc = read(fd, (char *)buf + cc, count);
			if (ncc <= 0)
				break;
			cc += ncc;
			count -= ncc;
		}
		return cc;
	}

	if (!forcereads || cc >= 0)
		return cc;

	assert(fd != infd || incanseek);

	/*
	 * Got an error reading the range.
	 * Retry one sector at a time, replacing any bad sectors with
	 * zeroed data.
	 */
	if ((startoffset = lseek(fd, (off_t) 0, SEEK_CUR)) < 0) {
		perror("devread: seeking to get input file ptr");
		exit(1);
	}
	assert((startoffset & (DEV_BSIZE-1)) == 0);

	onbytes = nbytes;
	while (nbytes > 0) {
		if (nbytes > DEV_BSIZE)
			count = DEV_BSIZE;
		else
			count = nbytes;
		cc = slowread(fd, buf, count, startoffset);
		if (cc != count) {
			fprintf(stderr, "devread: read failed on sector %u, "
				"returning zeros\n",
				bytestosec(startoffset));
			if (cc < 0)
				cc = 0;
			memset(buf, 0, count-cc);
			badsectors++;
			cc = count;
		}
		nbytes -= cc;
		buf = (void *)((char *)buf + cc);
		startoffset += cc;
	}
	return onbytes;
}

/*
 * Wrap up write in a retry mechanism to protect against transient NFS
 * errors causing a fatal error.
 */
ssize_t
devwrite(int fd, const void *buf, size_t nbytes)
{
	int		cc, i, count = 0;
	off_t		startoffset = 0;

	if (retrywrites && outcanseek &&
	    ((startoffset = lseek(fd, (off_t) 0, SEEK_CUR)) < 0)) {
		perror("devwrite: seeking to get output file ptr");
		exit(1);
	}

	for (i = 0; i < WRITE_RETRIES; i++) {
		while (nbytes) {
			cc = write(fd, buf, nbytes);

			if (cc > 0) {
				nbytes -= cc;
				buf     = (void *)((char *)buf + cc);
				count  += cc;
				continue;
			}

			if (!retrywrites)
				return cc;

			if (i == 0)
				perror("write error: will retry");

			sleep(1);
			nbytes += count;
			buf     = (void *)((char *)buf - count);
			count   = 0;
			goto again;
		}
		if (retrywrites && fsync(fd) < 0) {
			perror("fsync error: will retry");
			sleep(1);
			nbytes += count;
			buf     = (void *)((char *)buf - count);
			count   = 0;
			goto again;
		}
		datawritten += count;
		return count;
	again:
		if (lseek(fd, startoffset, SEEK_SET) < 0) {
			perror("devwrite: seeking to set file ptr");
			exit(1);
		}
	}
	perror("write error: busted for too long");
	fflush(stderr);
	exit(1);
}

static int
setpartition(partmap_t map, char *str)
{
	int dospart;
	char bsdpart;

	if (isdigit(str[1])) {
		bsdpart = str[2];
		str[2] = '\0';
	} else {
		bsdpart = str[1];
		str[1] = '\0';
	}
	dospart = atoi(str);
	if (dospart < 1 || dospart > MAXSLICES)
		return EINVAL;

	/* common case: apply to complete DOS partition */
	if (bsdpart == '\0') {
		map[dospart-1] = ~0;
		return 0;
	}

	if (bsdpart < 'a' || bsdpart > 'p')
		return EINVAL;

	map[dospart-1] |= (1 << (bsdpart - 'a'));
	return 0;
}

int
main(int argc, char *argv[])
{
	int	ch, rval;
	char	*outfilename = 0;
	int	rawmode	  = 0;
	int	slicetype = 0;
	struct timeval sstamp;
	extern char build_info[];

	memset(imageid, '\0', UUID_LENGTH);

	gettimeofday(&sstamp, 0);
	while ((ch = getopt(argc, argv, "vlbnNdihrs:c:z:ofI:13F:DR:S:XxH:U:P:Me:k:u:a:Z")) != -1)
		switch(ch) {
		case 'v':
			version++;
			break;
		case 'i':
			info++;
			debug++;
			break;
		case 'D':
			retrywrites = 0;
			break;
		case 'd':
			debug++;
			break;
		case 'l':
			slicetype = IZTYPE_LINUX;
			break;
		case 'b':
			slicetype = IZTYPE_FBSDNOLABEL;
			break;
		case 'N':
			dorelocs = 0;
			break;
		case 'n':
			slicetype = IZTYPE_NTFS;
			break;
		case 'o':
			dots++;
			break;
		case 'r':
			rawmode++;
			break;
		case 'S':
			slicetype = atoi(optarg);
			/* XXX special case for freebsd */
			if (slicetype == IZTYPE_386BSD)
				slicetype = IZTYPE_FBSDNOLABEL;
			break;
		case 's':
			slicemode = 1;
			slice = atoi(optarg);
			break;
		case 'z':
			level = atoi(optarg);
			if (level < 0 || level > 9)
				usage();
			break;
		case 'c':
			maxmode     = 1;
			inputmaxsec = atoi(optarg);
			break;
		case 'I':
			if (setpartition(ignore, optarg))
				usage();
			break;
		case 'R':
			if (setpartition(forceraw, optarg))
				usage();
			break;
		case '1':
			compat = COMPRESSED_V1;
			break;
		case '3':
			compat = COMPRESSED_V3;
			break;
		case 'F':
			frangesize = atoi(optarg);
			if (frangesize < 0)
				usage();
			break;
		case 'Z':
			zerofrange = 1;
			break;
		case 'x':
			excludenonfs++;
			break;
		case 'X':
			forcereads++;
			break;
#ifdef WITH_HASH
		case 'H':
			hashfile = optarg;
			break;
		case 'U':
			newhashfile = optarg;
			break;
		case 'P':
			deltapct = atoi(optarg);
			if (deltapct < 0)
				usage();
			break;
#else
		case 'H':
		case 'U':
		case 'P':
			fprintf(stderr, "'%c' option not supported\n", ch);
			usage();
			break;
#endif
		case 'M':
			metaoptimize++;
			break;
		case 'f':
			filemode++;
			rawmode++;
			break;
		case 'a':
#ifdef WITH_CRYPTO
			/* Authentication (checksum) hash algorithm */
			if (strcmp(optarg, "SHA1") == 0) {
				csumalg = CSUM_SHA1;
			}
			else {
				fprintf(stderr, "Only know \"SHA1\"\n");
				usage();
			}
			do_checksum++;
#ifdef SIGN_CHECKSUM
			csumalg |= CSUM_SIGNED;
#else
			fprintf(stderr, "WARNING: checksum is not signed\n");
#endif
#else
			fprintf(stderr, "'a' option not supported\n");
			usage();
#endif
			break;

		case 'e':
#ifdef WITH_CRYPTO
			/* Encryption cipher */
			if (strcmp(optarg, "bf_cbc") == 0) {
				cipher = ENC_BLOWFISH_CBC;
			}
			else {
				fprintf(stderr,
					"Only know \"bf_cbc\" (blowfish CBC)\n");
				usage();
			}
			do_encrypt++;
#else
			fprintf(stderr, "'e' option not supported\n");
			usage();
#endif
			break;
		case 'k':
#ifdef WITH_CRYPTO
			/* Encryption key file */
			enc_key = calloc(1, ENC_MAX_KEYLEN);
			if (enc_key == NULL ||
			    !encrypt_readkey(optarg, enc_key, ENC_MAX_KEYLEN))
				usage();
			/* XXX can you intuit the cipher from the key? */
			if (cipher == ENC_NONE)
				cipher = ENC_BLOWFISH_CBC;
#else
			fprintf(stderr, "'k' option not supported\n");
			usage();
#endif
			break;
		case 'u':
			/* UUID for image id. */
			if (!hexstr_to_mem(imageid, optarg, UUID_LENGTH))
				usage();
			got_imageid = 1;
			break;
		case 'h':
		case '?':
		default:
			usage();
		}
	argc -= optind;
	argv += optind;

	if (version || debug) {
		fprintf(stderr, "%s\n", build_info);
		if (version) {
			char *str;

			fprintf(stderr, "Supports:\n");
			fprintf(stderr, "  Partitioning: ");
#ifdef WITH_MBR
			fprintf(stderr, "MBR");
			str = ",";
#endif
#ifdef WITH_GPT
			fprintf(stderr, "%sGPT", str);
#endif
			fprintf(stderr, "\n  Partition types: ");
			printslicemap();
			fprintf(stderr, "\n  Features: ");
			fprintf(stderr, "image UUIDs");
#ifdef WITH_CRYPTO
#ifdef SIGN_CHECKSUM
			fprintf(stderr, ",signed");
#else
			fprintf(stderr, ",unsigned");
#endif
			fprintf(stderr, " SHA1 checksums");
			fprintf(stderr, ",blowfish encryption");
#endif
#ifdef WITH_HASH
			fprintf(stderr, ",delta image creation");
#endif
			fprintf(stderr, "\n");
			exit(0);
		}
	}

	if (argc < 1 || argc > 2)
		usage();

	if (compat &&
	    (
#ifdef WITH_CRYPTO
	     do_encrypt || do_checksum ||
#endif
	     got_imageid)) {
		fprintf(stderr, "Cannot use uuid/encrypt/checksum with -3\n");
		usage();
	}
	if (slicemode && (slice < 1 || slice > MAXSLICES)) {
		fprintf(stderr, "Slice must be a DOS partition (1-4) "
			"or extended DOS partition (5-%d)\n\n", MAXSLICES);
		usage();
	}
	if (maxmode && slicemode) {
		fprintf(stderr, "Count option (-c) cannot be used with "
			"the slice (-s) option\n\n");
		usage();
	}
	if (!info && argc != 2) {
		fprintf(stderr, "Must specify an output filename!\n\n");
		usage();
	}
	else
		outfilename = argv[1];

	if (!slicemode && !filemode && dorelocs)
		dorelocs = 0;

	infilename = argv[0];
	if (strcmp(infilename, "-")) {
		if ((infd = open(infilename, O_RDONLY, 0)) < 0) {
			perror(infilename);
			exit(1);
		}
		incanseek = 1;
	}
	else {
		if (!rawmode) {
			fprintf(stderr,
				"Can only use stdin as input with -f or -r\n");
			usage();
		}
		infd = fileno(stdin);
		incanseek = 0;
		forcereads = 0;
	}

#ifdef WITH_CRYPTO
	/*
	 * Let's get random!
	 */
	if (!RAND_load_file("/dev/urandom", 1024))
		fprintf(stderr, "Error getting random seed\n");

#ifdef SIGN_CHECKSUM
	/*
	 * Generate a signing pubkey.
	 * The pubkey is written to <imagename>.skey.
	 */
	if (do_checksum) {
		sig_key = RSA_generate_key(CSUM_MAX_LEN*8, 17, NULL, NULL);
		if (!info)
			output_public_key(outfilename, sig_key);
	}
#endif

	/*
	 * Generate an encryption key if none was given on the command line.
	 * The key is written to <imagename>.ekey.
	 */
	if (do_encrypt && enc_key == NULL) {
		enc_key = calloc(1, ENC_MAX_KEYLEN);
		if (enc_key == NULL || !RAND_bytes(enc_key, ENC_MAX_KEYLEN)) {
			fprintf(stderr, "Unable to generate random key\n");
			exit(1);
		}
		if (!info)
			output_encrypt_key(outfilename, enc_key,
					   ENC_MAX_KEYLEN);
	}
#endif

#ifdef WITH_V3COMPAT
	/*
	 * Deal with pre-crypto backward compatibility.
	 * We don't want to generate V4 images unless we really need it.
	 * Note that this means we do NOT generate a UUID if none was provided.
	 */
	if (!compat &&
#ifdef WITH_CRYPTO
	    !do_encrypt && !do_checksum &&
#endif
	    !got_imageid)
		compat = COMPRESSED_V3;
#endif

	/*
	 * Generate a random UUID if one was not provided and we are
	 * not operating in compatibility mode.
	 */
	if (!compat && !got_imageid) {
		int fd = open("/dev/urandom", O_RDONLY, 0);

		if (fd < 0 ||
		    read(fd, imageid, sizeof(imageid)) != sizeof(imageid)) {
			fprintf(stderr, "WARNING: no UUID generated\n");
			memset(imageid, '\0', UUID_LENGTH);
		} else {
			char uuidstr[UUID_LENGTH*2+1];

			mem_to_hexstr(uuidstr, imageid, UUID_LENGTH);
			output_uuid(outfilename, uuidstr);
			got_imageid = 1;
		}

		if (fd >= 0)
			close(fd);
	}

	/*
	 * Create the skip list by scanning the filesystems on
	 * the disk or indicated partition.
	 */
	if (slicetype != 0) {
		/*
		 * See if we can get a meaningful max size value via seek
		 */
		if (inputmaxsec == 0) {
			off_t maxoff = lseek(infd, (off_t)0, SEEK_END);
			if (maxoff != (off_t)-1 && maxoff != (off_t)0) {
				inputmaxsec = bytestosec(maxoff);
				assert(inputmaxsec == 0 ||
				       inputmaxsec > inputminsec);
				if (debug)
					fprintf(stderr, "Max sector: %lu\n",
						inputmaxsec);
			}
			(void) lseek(infd, (off_t)0, SEEK_SET);
		}

		rval = read_slice(-1, slicetype, 0, inputmaxsec,
				  infilename, infd);
		if (rval == -1)
			fprintf(stderr, ", cannot process\n");
	} else if (rawmode)
		rval = read_raw(infd);
	else
		rval = read_image(infd);
	if (rval) {
		fprintf(stderr, "* * * Aborting * * *\n");
		exit(1);
	}

	/*
	 * Create a valid range list from the skip list
	 */
	(void) mergeskips(debug > 1);
	if (debug)
		dumpskips(debug > 1);
	makeranges();
	if (debug)
		dumpranges(debug > 1);
	if (ranges == NULL) {
		/*
		 * No valid ranges, exit with an error.
		 *
		 * XXX it is debatable whether this is really an error.
		 * Perhaps we should just create a valid image file with
		 * one chunk containing no valid ranges, but it seems likely
		 * that they probably made a mistake and we should call it
		 * out immediately.
		 */
		fprintf(stderr,
			"*** No valid data on specified disk/partition!?\n");
		exit(1);
	}
	sortrange(&fixups, 0, cmpfixups);
	if (debug > 1)
		dumpfixups(debug > 2, 0);
	fflush(stderr);

#ifdef WITH_HASH
	/*
	 * If we are creating a "delta" image from a hash signature,
	 * (hashfile != NULL) we read in the signature info and reconcile
	 * that with the known allocated range that we have just computed.
	 * The result is a new list of ranges that are currently allocated
	 * and that have changed from the signature version.
	 *
	 * If we are creating a new signature file (newhashfile != NULL)
	 * then we also collect hashinfo along the way, writing out the
	 * newfile when done.
	 */
	if (hashfile || newhashfile) {
		struct range *nranges = NULL;

		/*
		 * next compare allocated 'ranges' and 'hinfo' to find out the
		 * changed blocks -- computing the hashes for some 'ranges'
		 * in the process
		 */
		if (hashmap_compute_delta(ranges, hashfile, infd, inputminsec,
					  newhashfile, &nranges)) {
			fprintf(stderr, "Errors while computing delta!\n");
			exit(1);
		}

		/*
		 * See if we should just create a full image.
		 */
		if (hashfile && deltapct >= 0) {
			uint32_t oldsect = sectinranges(ranges);
			uint32_t newsect = sectinranges(nranges);
			int dofull =
				(oldsect == 0 ||
				 ((double)newsect / oldsect) * 100 > deltapct);
			fprintf(stderr,
				"Full image size %u sect, "
				"delta image size %u sect\n"
				"Auto image selection creating %s image.\n",
				oldsect, newsect, dofull ? "full" : "delta");
			if (dofull) {
				freeranges(nranges);
				goto done;
			}
		}

		freeranges(ranges);
		ranges = nranges;
		if (debug) {
			fprintf(stderr, "\nAfter delta computation: ");
			dumpranges(debug > 1);
			dumpfixups(debug > 2, 0);
		}
	done:
		hashmap_dump_stats(slice);
	}
#endif

	/*
	 * Now we have all the allocated information, create the image
	 * (unless we just want an info report).
	 */
	if (!info) {
		if (strcmp(outfilename, "-")) {
			if ((outfd = open(outfilename, O_RDWR|O_CREAT|O_TRUNC,
					  0666)) < 0) {
				perror("opening output file");
				exit(1);
			}
			outcanseek = 1;
			/*
			 * XXX retrywrites is a hack to avoid transient NFS
			 * write errors: we use fsync after every write to
			 * check that it worked, re-trying if not. However,
			 * some devices (e.g., /dev/null) do not support
			 * fsync, so check for that here and disable the
			 * retry if necessary.
			 */
			if (retrywrites &&
			    fsync(outfd) < 0 && errno == EINVAL) {
				fprintf(stderr,
					"WARNING: '%s' does not support fsync,"
					" write errors may not be detected "
					"or corrected.\n", outfilename);
				retrywrites = 0;
			}
		}
		else {
			outfd = fileno(stdout);
			outcanseek = 0;
			retrywrites = 0;
		}
		compress_image();
		assert(fixups == NULL);

		if (outcanseek)
			close(outfd);

#ifdef WITH_HASH
		/*
		 * Write out new signature file.
		 */
		if (newhashfile &&
		    hashmap_write_hashfile(newhashfile, outfilename))
			fprintf(stderr, "Could not write new hashfile!\n");
#endif
	}
	close(infd);

	{
		struct timeval stamp;
		unsigned int ms;

		gettimeofday(&stamp, 0);
		if (stamp.tv_usec < sstamp.tv_usec) {
			stamp.tv_usec += 1000000;
			stamp.tv_sec--;
		}
		ms = (stamp.tv_sec - sstamp.tv_sec) * 1000 +
			(stamp.tv_usec - sstamp.tv_usec) / 1000;
		fprintf(stderr,
			"Finished in %u.%03u seconds\n",
			ms / 1000, ms % 1000);
	}
	fflush(stderr);

#ifdef WITH_CRYPTO
#ifdef SIGN_CHECKSUM
	if (do_checksum)
		RSA_free(sig_key);
#endif
#endif

	exit(0);
}

static int
read_slice(int snum, iz_type stype, iz_lba start, iz_size size,
	   char *sname, int sfd)
{
	struct sliceinfo *smap = getslicemap(stype);

	if (smap && smap->process)
		return (*smap->process)(snum, stype, start, size, sname, sfd);

	fprintf(stderr, "Slice %d is an unknown type %#x (%s)",
		snum+1, stype, smap ? smap->desc : "??");
	return -1;
}

/*
 * Parse the MBR/GPT and dispatch to the individual readers.
 */
int
read_image(int fd)
{
	int		i, rval = 0;
	struct iz_disk	disk;
	struct iz_slice	*parttab;
	int		gotbb = 0;
	char *		bbstr;
	iz_lba		dstart;
	iz_size		dsize;
	int		dowarn = debug ? 2 : 0;

#ifdef WITH_GPT
	if (!gotbb && parse_gpt(fd, &disk, dowarn) == 0) {
		gotbb = 1;
		bbstr = "GPT";
	}
#endif
#ifdef WITH_MBR
	if (!gotbb && parse_mbr(fd, &disk, dowarn) == 0) {
		gotbb = 2;
		bbstr = "MBR";
	}
	/*
	 * If we are forcing interpretation as a BSD slice, there should
	 * not be an MBR, however there is sometimes this thing called the
	 * "historical bogus partition table" (by the geom code) which
	 * is an MBR with P4 having 50000 sectors. We need to avoid this
	 * so we force looking for just a disklabel instead.
	 */
	if (gotbb == 2) {
		parttab = disk.slices;
		if (parttab[3].type == IZTYPE_386BSD &&
		    parttab[3].size == 50000 && parttab[3].offset == 0 &&
		    parttab[2].size == 0 && parttab[2].offset == 0 &&
		    parttab[1].size == 0 && parttab[1].offset == 0 &&
		    parttab[0].size == 0 && parttab[0].offset == 0) {
			fprintf(stderr,
				"WARNING: ignoring historical bogus MBR\n");
			gotbb = 0;
		}
	}
#endif

	if (!gotbb) {
		/* try to parse as one of the FS types */
		warnx("No GPT or MBR partition table found!");
		return 1;
	}

	parttab = disk.slices;
	dstart = 0;
	dsize = disk.dsize;

	if (debug) {
		int i;

		fprintf(stderr, "Disk:            start %12d, size %12d\n",
			0, dsize);
		fprintf(stderr, "Usable:          start %12d, size %12d\n",
			disk.lodata, disk.hidata - disk.lodata + 1);
		fprintf(stderr, "Partition range: start %12d, size %12d\n",
			disk.losect, disk.hisect - disk.losect + 1);
		fprintf(stderr, "%s Partitions:\n", bbstr);
		for (i = 0; i < MAXSLICES; i++) {
			struct sliceinfo *sinfo;

			if (parttab[i].type == IZTYPE_INVALID)
				continue;

			fprintf(stderr, "  P%d: ", i+1);
			sinfo = getslicemap(parttab[i].type);
			if (sinfo == 0)
				fprintf(stderr, "0x%x", parttab[i].type);
			else
				fprintf(stderr, "%-12s", sinfo->desc);

			fprintf(stderr, "  start %12d, size %12d",
				parttab[i].offset, parttab[i].size);
			if (parttab[i].flags) {
				fprintf(stderr, " (");
				if (parttab[i].flags & IZFLAG_NOTSUP)
					fprintf(stderr, "Not supported,");
				if (parttab[i].flags & IZFLAG_IGNORE)
					fprintf(stderr, "IGNORED,");
				if (parttab[i].flags & IZFLAG_RAW)
					fprintf(stderr, "compress RAW");
				fprintf(stderr, ")\n");
			} else {
				fprintf(stderr, "\n");
			}
		}
		fprintf(stderr, "\n");
	}

	/*
	 * Quick, brute-force check for overlap of partitions.
	 * XXX right now, any overlap is bad and we bail.  In the future,
	 * we could determine all areas of intersection and be conservative
	 * with those areas; i.e., always save unless overlap is strictly
	 * between unused/ignored partitions.
	 */
	for (i = 0; i < MAXSLICES; i++) {
		iz_lba start1, start2;
		iz_size size1, size2;
		int ii;

		if (parttab[i].type == IZTYPE_INVALID)
			continue;

		if ((size1 = parttab[i].size) == 0)
			continue;

		start1 = parttab[i].offset;
		for (ii = i + 1; ii < MAXSLICES; ii++) {
			if (parttab[ii].type == IZTYPE_INVALID)
				continue;
			if ((size2 = parttab[ii].size) == 0)
				continue;
			start2 = parttab[ii].offset;
			if (start2+size2 > start1 &&
			    start1+size1 > start2) {
				warnx("P%d and P%d overlap!", i+1, ii+1);
				rval++;
			}
		}
	}
	if (rval)
		return 1;

	/*
	 * If not in slice mode, skip sectors outside of any partition.
	 *
	 * XXX we could skip space between partitions too since we don't
	 * allow overlapping partitions. But partitions don't have to be
	 * in lowest to highest order on the disk, so it is harder.
	 */
	if (!slicemode) {
		iz_lba dlow = disk.lodata;
		iz_lba dhigh = maxmode ? (inputmaxsec-1) : disk.hidata;
		iz_lba losect = disk.losect;
		iz_lba hisect = disk.hisect;

		if (losect > dlow) {
			addskip(dlow, losect-dlow);
			if (dowarn > 1)
				warnx("%s: skipping %u sectors at %u",
				      bbstr, losect - dlow, dlow);
		}
		if (hisect < dhigh) {
			addskip(hisect+1, dhigh-hisect);
			if (dowarn > 1)
				warnx("%s: skipping %lu sectors at %u",
				      bbstr, (unsigned long)(dhigh - hisect),
				      hisect + 1);
		}
	}

	/*
	 * Now operate on individual slices.
	 */
	for (i = 0; i < MAXSLICES; i++) {
		iz_type	type  = parttab[i].type;
		iz_lba start  = parttab[i].offset;
		iz_size size  = parttab[i].size;
		iz_flags flags= parttab[i].flags;

		/*
		 * No such slice
		 */
		if (type == IZTYPE_INVALID)
			continue;

		/*
		 * This is not the slice you are looking for...
		 */
		if (slicemode && i + 1 != slice)
			continue;

		/*
		 * Explicit request by the user to ignore the partition
		 */
		if (ignore[i] && (!ISBSD(type) || ignore[i] == ~0)) {
			fprintf(stderr,
				"P%d: forcing skip, NOT SAVING.\n", i+1);
			flags |= IZFLAG_IGNORE;
		}
		/*
		 * Explicit request by the user to compress raw
		 */
		else if (forceraw[i] && (!ISBSD(type) || forceraw[i] == ~0)) {
			fprintf(stderr,
				"P%d: forcing raw compression.\n", i+1);
			flags |= IZFLAG_RAW;
		}
		/*
		 * Partition type is unsupported. We could either compress
		 * it raw or ignore it. Right now we always compress them
		 * raw. We could add an option to ignore them.
		 */
		else if ((flags & IZFLAG_NOTSUP) != 0) {
			fprintf(stderr,
				"P%d: Type 0x%x not supported, ", i+1, type);

			fprintf(stderr, "forcing raw compression.\n");
			flags |= IZFLAG_RAW;
		}
		/*
		 * Partition is being ignored either because it is unused
		 * or otherwise contains no useful info (e.g., extended DOS
		 * partitions).
		 */
		else if ((flags & IZFLAG_IGNORE) != 0) {
			fprintf(stderr,
				"P%d: %s, NOT SAVING.\n", i+1,
				type == IZTYPE_UNUSED ? "unused" : "ignored");
			if (size > 0)
				addskip(start, size);
		}
		/*
		 * Finally! We have a partition that we can apply some
		 * smarts to. If that fails, we try raw compression.
		 */
		else {
			rval = read_slice(i, type, start, size,
					  infilename, fd);
			if (rval == -1) {
				fprintf(stderr, ", forcing raw compression\n");
				rval = 0;
			}
		}

		/*
		 * In slicemode, we need to set the bounds of compression.
		 * If not in slicemode, we cannot set the bounds according
		 * to the partition info since its possible that someone
		 * will create a disk with empty space before the first
		 * partition (typical, to start partition 1 at the second
		 * cylinder) or after the last partition (Mike!). However,
		 * do not set the inputminsec since we usually want the
		 * stuff before the first partition, which is the boot stuff.
		 */
		if (slicemode) {
			inputminsec = start;
			inputmaxsec = start + size;
		} else if (!maxmode && (start + size) > inputmaxsec) {
			inputmaxsec = start + size;
		}
	}

	/*
	 * Ultimately, we are going to believe the partitioner about
	 * min/max values, but lets see how they compare.
	 */
	if (!slicemode) {
		if (dstart != inputminsec) {
			fprintf(stderr,
				"partitioner low value (%lu) different than "
				"computed value (%lu); using the former\n",
				(unsigned long)dstart, inputminsec);
			inputminsec = dstart;
		}
		if (!maxmode && dstart+dsize != inputmaxsec) {
			fprintf(stderr,
				"partitioner high value (%lu) different than "
{'fraction_non_alphanumeric': 0.1410455903131763, 'fraction_numerical': 0.012537783262685339, 'mean_word_length': 3.066918824556299, 'pattern_counts': {'":': 0, '<': 105, '<?xml version=': 0, '>': 117, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4207259', 'n_tokens_mistral': 24017, 'n_tokens_neox': 20836, 'n_words': 6359}