<commit_before>/*
 * Copyright (c) <NAME> 1986-1995.
 * Software written by <NAME> and others;
 * maintained 1995-present by <NAME> and others.
 * 
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice immediately at the beginning of the file, without modification,
 *    this list of conditions, and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *  
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */
/*
 * apprentice - make one pass through /etc/magic, learning its secrets.
 */

#include "file.h"
#include "magic.h"
#include <stdlib.h>
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#include <string.h>
#include <ctype.h>
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/param.h>
#ifdef QUICK
#include <sys/mman.h>
#endif

#ifndef	lint
FILE_RCSID("@(#)$Id: apprentice.c,v 1.87 2006/03/02 22:08:57 christos Exp $")
#endif	/* lint */

#define	EATAB {while (isascii((unsigned char) *l) && \
		      isspace((unsigned char) *l))  ++l;}
#define LOWCASE(l) (isupper((unsigned char) (l)) ? \
			tolower((unsigned char) (l)) : (l))
/*
 * Work around a bug in headers on Digital Unix.
 * At least confirmed for: OSF1 V4.0 878
 */
#if defined(__osf__) && defined(__DECC)
#ifdef MAP_FAILED
#undef MAP_FAILED
#endif
#endif

#ifndef MAP_FAILED
#define MAP_FAILED (void *) -1
#endif

#ifndef MAP_FILE
#define MAP_FILE 0
#endif

#ifndef MAXPATHLEN
#define MAXPATHLEN	1024
#endif

#define IS_PLAINSTRING(t) ((t) == FILE_STRING || (t) == FILE_PSTRING || \
    (t) == FILE_BESTRING16 || (t) == FILE_LESTRING16)
    
#define IS_STRING(t) (IS_PLAINSTRING(t) || (t) == FILE_REGEX || \
    (t) == FILE_SEARCH)

struct magic_entry {
	struct magic *mp;	
	uint32_t cont_count;
	uint32_t max_count;
};

private int getvalue(struct magic_set *ms, struct magic *, const char **);
private int hextoint(int);
private const char *getstr(struct magic_set *, const char *, char *, int,
    int *);
private int parse(struct magic_set *, struct magic_entry **, uint32_t *,
    const char *, int);
private void eatsize(const char **);
private int apprentice_1(struct magic_set *, const char *, int, struct mlist *);
private size_t apprentice_magic_strength(const struct magic *);
private int apprentice_sort(const void *, const void *);
private int apprentice_file(struct magic_set *, struct magic **, uint32_t *,
    const char *, int);
private void byteswap(struct magic *, uint32_t);
private void bs1(struct magic *);
private uint16_t swap2(uint16_t);
private uint32_t swap4(uint32_t);
private char *mkdbname(const char *, char *, size_t, int);
private int apprentice_map(struct magic_set *, struct magic **, uint32_t *,
    const char *);
private int apprentice_compile(struct magic_set *, struct magic **, uint32_t *,
    const char *);
private int check_format(struct magic_set *, struct magic *);

private size_t maxmagic = 0;
private size_t magicsize = sizeof(struct magic);


#ifdef COMPILE_ONLY

int main(int, char *[]);

int
main(int argc, char *argv[])
{
	int ret;
	struct magic_set *ms;
	char *progname;

	if ((progname = strrchr(argv[0], '/')) != NULL)
		progname++;
	else
		progname = argv[0];

	if (argc != 2) {
		(void)fprintf(stderr, "Usage: %s file\n", progname);
		return 1;
	}

	if ((ms = magic_open(MAGIC_CHECK)) == NULL) {
		(void)fprintf(stderr, "%s: %s\n", progname, strerror(errno));
		return 1;
	}
	ret = magic_compile(ms, argv[1]) == -1 ? 1 : 0;
	if (ret == 1)
		(void)fprintf(stderr, "%s: %s\n", progname, magic_error(ms));
	magic_close(ms);
	return ret;
}
#endif /* COMPILE_ONLY */


/*
 * Handle one file.
 */
private int
apprentice_1(struct magic_set *ms, const char *fn, int action,
    struct mlist *mlist)
{
	struct magic *magic = NULL;
	uint32_t nmagic = 0;
	struct mlist *ml;
	int rv = -1;
	int mapped;

	if (magicsize != FILE_MAGICSIZE) {
		file_error(ms, 0, "magic element size %lu != %lu",
		    (unsigned long)sizeof(*magic),
		    (unsigned long)FILE_MAGICSIZE);
		return -1;
	}

	if (action == FILE_COMPILE) {
		rv = apprentice_file(ms, &magic, &nmagic, fn, action);
		if (rv != 0)
			return -1;
		rv = apprentice_compile(ms, &magic, &nmagic, fn);
		free(magic);
		return rv;
	}

#ifndef COMPILE_ONLY
	if ((rv = apprentice_map(ms, &magic, &nmagic, fn)) == -1) {
		if (ms->flags & MAGIC_CHECK)
			file_magwarn(ms, "using regular magic file `%s'", fn);
		rv = apprentice_file(ms, &magic, &nmagic, fn, action);
		if (rv != 0)
			return -1;
		mapped = 0;
	}

	if (rv == -1)
		return rv;
	mapped = rv;
	     
	if (magic == NULL || nmagic == 0) {
		file_delmagic(magic, mapped, nmagic);
		return -1;
	}

	if ((ml = malloc(sizeof(*ml))) == NULL) {
		file_delmagic(magic, mapped, nmagic);
		file_oomem(ms);
		return -1;
	}

	ml->magic = magic;
	ml->nmagic = nmagic;
	ml->mapped = mapped;

	mlist->prev->next = ml;
	ml->prev = mlist->prev;
	ml->next = mlist;
	mlist->prev = ml;

	return 0;
#endif /* COMPILE_ONLY */
}

protected void
file_delmagic(struct magic *p, int type, size_t entries)
{
	if (p == NULL)
		return;
	switch (type) {
	case 2:
		p--;
		(void)munmap((void *)p, sizeof(*p) * (entries + 1));
		break;
	case 1:
		p--;
		/*FALLTHROUGH*/
	case 0:
		free(p);
		break;
	default:
		abort();
	}
}


/* const char *fn: list of magic files */
protected struct mlist *
file_apprentice(struct magic_set *ms, const char *fn, int action)
{
	char *p, *mfn, *afn = NULL;
	int file_err, errs = -1;
	struct mlist *mlist;

	if (fn == NULL)
		fn = getenv("MAGIC");
	if (fn == NULL)
		fn = MAGIC;

	if ((fn = mfn = strdup(fn)) == NULL) {
		file_oomem(ms);
		return NULL;
	}

	if ((mlist = malloc(sizeof(*mlist))) == NULL) {
		free(mfn);
		file_oomem(ms);
		return NULL;
	}
	mlist->next = mlist->prev = mlist;

	while (fn) {
		p = strchr(fn, PATHSEP);
		if (p)
			*p++ = '\0';
		if (*fn == '\0')
			break;
		if (ms->flags & MAGIC_MIME) {
			if ((afn = malloc(strlen(fn) + 5 + 1)) == NULL) {
				free(mfn);
				free(mlist);
				file_oomem(ms);
				return NULL;
			}
			(void)strcpy(afn, fn);
			(void)strcat(afn, ".mime");
			fn = afn;
		}
		file_err = apprentice_1(ms, fn, action, mlist);
		if (file_err > errs)
			errs = file_err;
		if (afn) {
			free(afn);
			afn = NULL;
		}
		fn = p;
	}
	if (errs == -1) {
		free(mfn);
		free(mlist);
		mlist = NULL;
		file_error(ms, 0, "could not find any magic files!");
		return NULL;
	}
	free(mfn);
	return mlist;
}

private size_t
apprentice_magic_strength(const struct magic *m)
{
	switch (m->type) {
	case FILE_BYTE:
		return 1;

	case FILE_SHORT:
	case FILE_LESHORT:
	case FILE_BESHORT:
		return 2;

	case FILE_LONG:
	case FILE_LELONG:
	case FILE_BELONG:
	case FILE_MELONG:
		return 4;

	case FILE_PSTRING:
	case FILE_STRING:
	case FILE_REGEX:
	case FILE_BESTRING16:
	case FILE_LESTRING16:
	case FILE_SEARCH:
		return m->vallen;

	case FILE_DATE:
	case FILE_LEDATE:
	case FILE_BEDATE:
	case FILE_MEDATE:
		return 4;

	case FILE_LDATE:
	case FILE_LELDATE:
	case FILE_BELDATE:
	case FILE_MELDATE:
		return 8;

	default:
		return 0;
	}
}

private int
apprentice_sort(const void *a, const void *b)
{
	const struct magic_entry *ma = a;
	const struct magic_entry *mb = b;
	size_t sa = apprentice_magic_strength(ma->mp);
	size_t sb = apprentice_magic_strength(mb->mp);
	if (sa == sb)
		return 0;
	else if (sa > sb)
		return -1;
	else
		return 1;
}

/*
 * parse from a file
 * const char *fn: name of magic file
 */
private int
apprentice_file(struct magic_set *ms, struct magic **magicp, uint32_t *nmagicp,
    const char *fn, int action)
{
	private const char hdr[] =
		"cont\toffset\ttype\topcode\tmask\tvalue\tdesc";
	FILE *f;
	char line[BUFSIZ+1];
	int errs = 0;
	struct magic_entry *marray;
	uint32_t marraycount, i, mentrycount;

	f = fopen(ms->file = fn, "r");
	if (f == NULL) {
		if (errno != ENOENT)
			file_error(ms, errno, "cannot read magic file `%s'",
			    fn);
		return -1;
	}

        maxmagic = MAXMAGIS;
	if ((marray = malloc(maxmagic * sizeof(*marray))) == NULL) {
		(void)fclose(f);
		file_oomem(ms);
		return -1;
	}
	marraycount = 0;

	/* print silly verbose header for USG compat. */
	if (action == FILE_CHECK)
		(void)fprintf(stderr, "%s\n", hdr);

	/* parse it */
	for (ms->line = 1; fgets(line, BUFSIZ, f) != NULL; ms->line++) {
		size_t len;
		if (line[0] == '#')	/* comment, do not parse */
			continue;
		len = strlen(line);
		if (len < 2) /* null line, garbage, etc */
			continue;
		if (line[len - 1] == '\n')
			line[len - 1] = '\0'; /* delete newline */
		if (parse(ms, &marray, &marraycount, line, action) != 0)
			errs++;
	}

	(void)fclose(f);
	if (errs)
		goto out;

#ifndef NOORDER
	qsort(marray, marraycount, sizeof(*marray), apprentice_sort);
#endif

	for (i = 0, mentrycount = 0; i < marraycount; i++)
		mentrycount += marray[i].cont_count;

	if ((*magicp = malloc(sizeof(**magicp) * mentrycount)) == NULL) {
		file_oomem(ms);
		errs++;
		goto out;
	}

	mentrycount = 0;
	for (i = 0; i < marraycount; i++) {
		(void)memcpy(*magicp + mentrycount, marray[i].mp,
		    marray[i].cont_count * sizeof(**magicp));
		mentrycount += marray[i].cont_count;
	}
out:
	for (i = 0; i < marraycount; i++)
		free(marray[i].mp);
	free(marray);
	if (errs) {
		*magicp = NULL;
		*nmagicp = 0;
		return errs;
	} else {
		*nmagicp = mentrycount;
		return 0;
	}

}

/*
 * extend the sign bit if the comparison is to be signed
 */
protected uint32_t
file_signextend(struct magic_set *ms, struct magic *m, uint32_t v)
{
	if (!(m->flag & UNSIGNED))
		switch(m->type) {
		/*
		 * Do not remove the casts below.  They are
		 * vital.  When later compared with the data,
		 * the sign extension must have happened.
		 */
		case FILE_BYTE:
			v = (char) v;
			break;
		case FILE_SHORT:
		case FILE_BESHORT:
		case FILE_LESHORT:
			v = (short) v;
			break;
		case FILE_DATE:
		case FILE_BEDATE:
		case FILE_LEDATE:
		case FILE_MEDATE:
		case FILE_LDATE:
		case FILE_BELDATE:
		case FILE_LELDATE:
		case FILE_MELDATE:
		case FILE_LONG:
		case FILE_BELONG:
		case FILE_LELONG:
		case FILE_MELONG:
			v = (int32_t) v;
			break;
		case FILE_STRING:
		case FILE_PSTRING:
		case FILE_BESTRING16:
		case FILE_LESTRING16:
		case FILE_REGEX:
		case FILE_SEARCH:
			break;
		default:
			if (ms->flags & MAGIC_CHECK)
			    file_magwarn(ms, "cannot happen: m->type=%d\n",
				    m->type);
			return ~0U;
		}
	return v;
}

/*
 * parse one line from magic file, put into magic[index++] if valid
 */
private int
parse(struct magic_set *ms, struct magic_entry **mentryp, uint32_t *nmentryp, 
    const char *line, int action)
{
	int i = 0;
	struct magic_entry *me;
	struct magic *m;
	const char *l = line;
	char *t;
	private const char *fops = FILE_OPS;
	uint32_t val;
	uint32_t cont_level, cont_count;

	cont_level = 0;

	while (*l == '>') {
		++l;		/* step over */
		cont_level++; 
	}

#define ALLOC_CHUNK	(size_t)10
#define ALLOC_INCR	(size_t)200

	if (cont_level != 0) {
		if (*nmentryp == 0) {
			file_error(ms, 0, "No current entry for continuation");
			return -1;
		}
		me = &(*mentryp)[*nmentryp - 1];
		if (me->cont_count == me->max_count) {
			struct magic *nm;
			size_t cnt = me->max_count + ALLOC_CHUNK;
			if ((nm = realloc(me->mp, sizeof(*nm) * cnt)) == NULL) {
				file_oomem(ms);
				return -1;
			}
			me->mp = m = nm;
			me->max_count = cnt;
		}
		m = &me->mp[me->cont_count++];
		memset(m, 0, sizeof(*m));
		m->cont_level = cont_level;
	} else {<commit_msg>Fix segfault when '/etc/malloc.conf@ -> AJ' due to differnce between calloc & malloc as agreed by vendor.  Also remove an unused variable.<commit_after>/*
 * Copyright (c) <NAME> 1986-1995.
 * Software written by <NAME> and others;
 * maintained 1995-present by <NAME> and others.
 * 
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice immediately at the beginning of the file, without modification,
 *    this list of conditions, and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *  
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */
/*
 * apprentice - make one pass through /etc/magic, learning its secrets.
 */

#include "file.h"
#include "magic.h"
#include <stdlib.h>
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#include <string.h>
#include <ctype.h>
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/param.h>
#ifdef QUICK
#include <sys/mman.h>
#endif

#ifndef	lint
FILE_RCSID("@(#)$Id: apprentice.c,v 1.87 2006/03/02 22:08:57 christos Exp $")
#endif	/* lint */

#define	EATAB {while (isascii((unsigned char) *l) && \
		      isspace((unsigned char) *l))  ++l;}
#define LOWCASE(l) (isupper((unsigned char) (l)) ? \
			tolower((unsigned char) (l)) : (l))
/*
 * Work around a bug in headers on Digital Unix.
 * At least confirmed for: OSF1 V4.0 878
 */
#if defined(__osf__) && defined(__DECC)
#ifdef MAP_FAILED
#undef MAP_FAILED
#endif
#endif

#ifndef MAP_FAILED
#define MAP_FAILED (void *) -1
#endif

#ifndef MAP_FILE
#define MAP_FILE 0
#endif

#ifndef MAXPATHLEN
#define MAXPATHLEN	1024
#endif

#define IS_PLAINSTRING(t) ((t) == FILE_STRING || (t) == FILE_PSTRING || \
    (t) == FILE_BESTRING16 || (t) == FILE_LESTRING16)
    
#define IS_STRING(t) (IS_PLAINSTRING(t) || (t) == FILE_REGEX || \
    (t) == FILE_SEARCH)

struct magic_entry {
	struct magic *mp;	
	uint32_t cont_count;
	uint32_t max_count;
};

private int getvalue(struct magic_set *ms, struct magic *, const char **);
private int hextoint(int);
private const char *getstr(struct magic_set *, const char *, char *, int,
    int *);
private int parse(struct magic_set *, struct magic_entry **, uint32_t *,
    const char *, int);
private void eatsize(const char **);
private int apprentice_1(struct magic_set *, const char *, int, struct mlist *);
private size_t apprentice_magic_strength(const struct magic *);
private int apprentice_sort(const void *, const void *);
private int apprentice_file(struct magic_set *, struct magic **, uint32_t *,
    const char *, int);
private void byteswap(struct magic *, uint32_t);
private void bs1(struct magic *);
private uint16_t swap2(uint16_t);
private uint32_t swap4(uint32_t);
private char *mkdbname(const char *, char *, size_t, int);
private int apprentice_map(struct magic_set *, struct magic **, uint32_t *,
    const char *);
private int apprentice_compile(struct magic_set *, struct magic **, uint32_t *,
    const char *);
private int check_format(struct magic_set *, struct magic *);

private size_t maxmagic = 0;
private size_t magicsize = sizeof(struct magic);


#ifdef COMPILE_ONLY

int main(int, char *[]);

int
main(int argc, char *argv[])
{
	int ret;
	struct magic_set *ms;
	char *progname;

	if ((progname = strrchr(argv[0], '/')) != NULL)
		progname++;
	else
		progname = argv[0];

	if (argc != 2) {
		(void)fprintf(stderr, "Usage: %s file\n", progname);
		return 1;
	}

	if ((ms = magic_open(MAGIC_CHECK)) == NULL) {
		(void)fprintf(stderr, "%s: %s\n", progname, strerror(errno));
		return 1;
	}
	ret = magic_compile(ms, argv[1]) == -1 ? 1 : 0;
	if (ret == 1)
		(void)fprintf(stderr, "%s: %s\n", progname, magic_error(ms));
	magic_close(ms);
	return ret;
}
#endif /* COMPILE_ONLY */


/*
 * Handle one file.
 */
private int
apprentice_1(struct magic_set *ms, const char *fn, int action,
    struct mlist *mlist)
{
	struct magic *magic = NULL;
	uint32_t nmagic = 0;
	struct mlist *ml;
	int rv = -1;
	int mapped;

	if (magicsize != FILE_MAGICSIZE) {
		file_error(ms, 0, "magic element size %lu != %lu",
		    (unsigned long)sizeof(*magic),
		    (unsigned long)FILE_MAGICSIZE);
		return -1;
	}

	if (action == FILE_COMPILE) {
		rv = apprentice_file(ms, &magic, &nmagic, fn, action);
		if (rv != 0)
			return -1;
		rv = apprentice_compile(ms, &magic, &nmagic, fn);
		free(magic);
		return rv;
	}

#ifndef COMPILE_ONLY
	if ((rv = apprentice_map(ms, &magic, &nmagic, fn)) == -1) {
		if (ms->flags & MAGIC_CHECK)
			file_magwarn(ms, "using regular magic file `%s'", fn);
		rv = apprentice_file(ms, &magic, &nmagic, fn, action);
		if (rv != 0)
			return -1;
		mapped = 0;
	}

	if (rv == -1)
		return rv;
	mapped = rv;
	     
	if (magic == NULL || nmagic == 0) {
		file_delmagic(magic, mapped, nmagic);
		return -1;
	}

	if ((ml = malloc(sizeof(*ml))) == NULL) {
		file_delmagic(magic, mapped, nmagic);
		file_oomem(ms);
		return -1;
	}

	ml->magic = magic;
	ml->nmagic = nmagic;
	ml->mapped = mapped;

	mlist->prev->next = ml;
	ml->prev = mlist->prev;
	ml->next = mlist;
	mlist->prev = ml;

	return 0;
#endif /* COMPILE_ONLY */
}

protected void
file_delmagic(struct magic *p, int type, size_t entries)
{
	if (p == NULL)
		return;
	switch (type) {
	case 2:
		p--;
		(void)munmap((void *)p, sizeof(*p) * (entries + 1));
		break;
	case 1:
		p--;
		/*FALLTHROUGH*/
	case 0:
		free(p);
		break;
	default:
		abort();
	}
}


/* const char *fn: list of magic files */
protected struct mlist *
file_apprentice(struct magic_set *ms, const char *fn, int action)
{
	char *p, *mfn, *afn = NULL;
	int file_err, errs = -1;
	struct mlist *mlist;

	if (fn == NULL)
		fn = getenv("MAGIC");
	if (fn == NULL)
		fn = MAGIC;

	if ((fn = mfn = strdup(fn)) == NULL) {
		file_oomem(ms);
		return NULL;
	}

	if ((mlist = malloc(sizeof(*mlist))) == NULL) {
		free(mfn);
		file_oomem(ms);
		return NULL;
	}
	mlist->next = mlist->prev = mlist;

	while (fn) {
		p = strchr(fn, PATHSEP);
		if (p)
			*p++ = '\0';
		if (*fn == '\0')
			break;
		if (ms->flags & MAGIC_MIME) {
			if ((afn = malloc(strlen(fn) + 5 + 1)) == NULL) {
				free(mfn);
				free(mlist);
				file_oomem(ms);
				return NULL;
			}
			(void)strcpy(afn, fn);
			(void)strcat(afn, ".mime");
			fn = afn;
		}
		file_err = apprentice_1(ms, fn, action, mlist);
		if (file_err > errs)
			errs = file_err;
		if (afn) {
			free(afn);
			afn = NULL;
		}
		fn = p;
	}
	if (errs == -1) {
		free(mfn);
		free(mlist);
		mlist = NULL;
		file_error(ms, 0, "could not find any magic files!");
		return NULL;
	}
	free(mfn);
	return mlist;
}

private size_t
apprentice_magic_strength(const struct magic *m)
{
	switch (m->type) {
	case FILE_BYTE:
		return 1;

	case FILE_SHORT:
	case FILE_LESHORT:
	case FILE_BESHORT:
		return 2;

	case FILE_LONG:
	case FILE_LELONG:
	case FILE_BELONG:
	case FILE_MELONG:
		return 4;

	case FILE_PSTRING:
	case FILE_STRING:
	case FILE_REGEX:
	case FILE_BESTRING16:
	case FILE_LESTRING16:
	case FILE_SEARCH:
		return m->vallen;

	case FILE_DATE:
	case FILE_LEDATE:
	case FILE_BEDATE:
	case FILE_MEDATE:
		return 4;

	case FILE_LDATE:
	case FILE_LELDATE:
	case FILE_BELDATE:
	case FILE_MELDATE:
		return 8;

	default:
		return 0;
	}
}

private int
apprentice_sort(const void *a, const void *b)
{
	const struct magic_entry *ma = a;
	const struct magic_entry *mb = b;
	size_t sa = apprentice_magic_strength(ma->mp);
	size_t sb = apprentice_magic_strength(mb->mp);
	if (sa == sb)
		return 0;
	else if (sa > sb)
		return -1;
	else
		return 1;
}

/*
 * parse from a file
 * const char *fn: name of magic file
 */
private int
apprentice_file(struct magic_set *ms, struct magic **magicp, uint32_t *nmagicp,
    const char *fn, int action)
{
	private const char hdr[] =
		"cont\toffset\ttype\topcode\tmask\tvalue\tdesc";
	FILE *f;
	char line[BUFSIZ+1];
	int errs = 0;
	struct magic_entry *marray;
	uint32_t marraycount, i, mentrycount;

	f = fopen(ms->file = fn, "r");
	if (f == NULL) {
		if (errno != ENOENT)
			file_error(ms, errno, "cannot read magic file `%s'",
			    fn);
		return -1;
	}

	maxmagic = MAXMAGIS;
	if ((marray = calloc(maxmagic, sizeof(*marray))) == NULL) {
		(void)fclose(f);
		file_oomem(ms);
		return -1;
	}
	marraycount = 0;

	/* print silly verbose header for USG compat. */
	if (action == FILE_CHECK)
		(void)fprintf(stderr, "%s\n", hdr);

	/* parse it */
	for (ms->line = 1; fgets(line, BUFSIZ, f) != NULL; ms->line++) {
		size_t len;
		if (line[0] == '#')	/* comment, do not parse */
			continue;
		len = strlen(line);
		if (len < 2) /* null line, garbage, etc */
			continue;
		if (line[len - 1] == '\n')
			line[len - 1] = '\0'; /* delete newline */
		if (parse(ms, &marray, &marraycount, line, action) != 0)
			errs++;
	}

	(void)fclose(f);
	if (errs)
		goto out;

#ifndef NOORDER
	qsort(marray, marraycount, sizeof(*marray), apprentice_sort);
#endif

	for (i = 0, mentrycount = 0; i < marraycount; i++)
		mentrycount += marray[i].cont_count;

	if ((*magicp = malloc(sizeof(**magicp) * mentrycount)) == NULL) {
		file_oomem(ms);
		errs++;
		goto out;
	}

	mentrycount = 0;
	for (i = 0; i < marraycount; i++) {
		(void)memcpy(*magicp + mentrycount, marray[i].mp,
		    marray[i].cont_count * sizeof(**magicp));
		mentrycount += marray[i].cont_count;
	}
out:
	for (i = 0; i < marraycount; i++)
		free(marray[i].mp);
	free(marray);
	if (errs) {
		*magicp = NULL;
		*nmagicp = 0;
		return errs;
	} else {
		*nmagicp = mentrycount;
		return 0;
	}

}

/*
 * extend the sign bit if the comparison is to be signed
 */
protected uint32_t
file_signextend(struct magic_set *ms, struct magic *m, uint32_t v)
{
	if (!(m->flag & UNSIGNED))
		switch(m->type) {
		/*
		 * Do not remove the casts below.  They are
		 * vital.  When later compared with the data,
		 * the sign extension must have happened.
		 */
		case FILE_BYTE:
			v = (char) v;
			break;
		case FILE_SHORT:
		case FILE_BESHORT:
		case FILE_LESHORT:
			v = (short) v;
			break;
		case FILE_DATE:
		case FILE_BEDATE:
		case FILE_LEDATE:
		case FILE_MEDATE:
		case FILE_LDATE:
		case FILE_BELDATE:
		case FILE_LELDATE:
		case FILE_MELDATE:
		case FILE_LONG:
		case FILE_BELONG:
		case FILE_LELONG:
		case FILE_MELONG:
			v = (int32_t) v;
			break;
		case FILE_STRING:
		case FILE_PSTRING:
		case FILE_BESTRING16:
		case FILE_LESTRING16:
		case FILE_REGEX:
		case FILE_SEARCH:
			break;
		default:
			if (ms->flags & MAGIC_CHECK)
			    file_magwarn(ms, "cannot happen: m->type=%d\n",
				    m->type);
			return ~0U;
		}
	return v;
}

/*
 * parse one line from magic file, put into magic[index++] if valid
 */
private int
parse(struct magic_set *ms, struct magic_entry **mentryp, uint32_t *nmentryp, 
    const char *line, int action)
{
	int i = 0;
	struct magic_entry *me;
	struct magic *m;
	const char *l = line;
	char *t;
	private const char *fops = FILE_OPS;
	uint32_t val;
	uint32_t cont_level;

	cont_level = 0;

	while (*l == '>') {
		++l;		/* step over */
		cont_level++; 
	}

#define ALLOC_CHUNK	(size_t)10
#define ALLOC_INCR	(size_t)200

	if (cont_level != 0) {
		if (*nmentryp == 0) {
			file_error(ms, 0, "No current entry for continuation");
			return -1;
		}
		me = &(*mentryp)[*nmentryp - 1];
		if (me->cont_count == me->max_count) {
			struct magic *nm;
			size_t cnt = me->max_count + ALLOC_CHUNK;
			if ((nm = realloc(me->mp, sizeof(*nm) * cnt)) == NULL) {
				file_oomem(ms);
				return -1;
			}
			me->mp = m = nm;
			me->max_count = cnt;
		}
		m = &me->mp[me->cont_count++];
		memset(m, 0, sizeof(*m));
		m->cont_level = cont_level;
	} else {
{'fraction_non_alphanumeric': 0.14242411995311804, 'fraction_numerical': 0.014630400517318029, 'mean_word_length': 3.6250467289719626, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 100, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4859908', 'n_tokens_mistral': 10386, 'n_tokens_neox': 9316, 'n_words': 2782}