<commit_before>/*
 * pmlogrewrite - config-driven stream editor for PCP archives
 *
 * Copyright (c) 1997-2002 Silicon Graphics, Inc.  All Rights Reserved.
 * Copyright (c) 2011 <NAME>.  All Rights Reserved.
 * 
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation; either version 2 of the License, or (at your
 * option) any later version.
 * 
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 */

#include <math.h>
#include <ctype.h>
#include <sys/stat.h>
#include "pmapi.h"
#include "impl.h"
#include "logger.h"
#include <assert.h>

global_t	global;
indomspec_t	*indom_root = NULL;
metricspec_t	*metric_root = NULL;
int		lineno;

/*
 *  Usage
 */
static void
usage(void)
{
    fprintf(stderr,
"Usage: %s [options] input-archive output-archive\n\
\n\
Options:\n\
  -c config   file or directory to load rules from\n\
  -C          parse config file(s) and quit\n\
  -d          desperate, save output archive even after error\n\
  -s          do scale conversion\n\
  -v          verbose\n\
  -w          emit warnings [default is silence]\n",
	pmProgname);
}

/*
 *  Global variables
 */
static int	first_datarec = 1;		/* first record flag */

off_t		new_log_offset;			/* new log offset */
off_t		new_meta_offset;		/* new meta offset */


/* archive control stuff */
inarch_t		inarch;		/* input archive control */
outarch_t		outarch;	/* output archive control */

/* command line args */
int	nconf = 0;			/* number of config files */
char	**conf = NULL;			/* list of config files */
char	*configfile = NULL;		/* current config file */
int	Cflag = 0;			/* -C parse config and quit */
int	dflag = 0;			/* -d desperate */
int	sflag = 0;			/* -s scale values */
int	vflag = 0;			/* -v verbosity */
int	wflag = 0;			/* -w emit warnings */

/*
 *  report that archive is corrupted
 */
static void
_report(FILE *fp)
{
    off_t	here;
    struct stat	sbuf;

    here = lseek(fileno(fp), 0L, SEEK_CUR);
    fprintf(stderr, "%s: Error occurred at byte offset %ld into a file of",
	    pmProgname, (long)here);
    if (fstat(fileno(fp), &sbuf) < 0)
	fprintf(stderr, ": stat: %s\n", osstrerror());
    else
	fprintf(stderr, " %ld bytes.\n", (long)sbuf.st_size);
    fprintf(stderr, "The last record, and the remainder of this file will not be extracted.\n");
    exit(1);
}

/*
 *  switch output volumes
 */
static void
newvolume(int vol)
{
    FILE		*newfp;

    if ((newfp = __pmLogNewFile(outarch.name, vol)) != NULL) {
	fclose(outarch.logctl.l_mfp);
	outarch.logctl.l_mfp = newfp;
	outarch.logctl.l_label.ill_vol = outarch.logctl.l_curvol = vol;
	__pmLogWriteLabel(outarch.logctl.l_mfp, &outarch.logctl.l_label);
	fflush(outarch.logctl.l_mfp);
    }
    else {
	fprintf(stderr, "%s: __pmLogNewFile(%s,%d) Error: %s\n",
		pmProgname, outarch.name, vol, pmErrStr(-oserror()));
	exit(1);
    }
}

/* construct new archive label */
static void
newlabel(void)
{
    __pmLogLabel	*lp = &outarch.logctl.l_label;

    /* copy magic number, pid, host and timezone */
    lp->ill_magic = inarch.label.ll_magic;
    lp->ill_pid = inarch.label.ll_pid;
    if (global.flags & GLOBAL_CHANGE_HOSTNAME)
	strncpy(lp->ill_hostname, global.hostname, PM_LOG_MAXHOSTLEN);
    else
	strncpy(lp->ill_hostname, inarch.label.ll_hostname, PM_LOG_MAXHOSTLEN);
    if (global.flags & GLOBAL_CHANGE_TZ)
	strncpy(lp->ill_tz, global.tz, PM_TZ_MAXLEN);
    else
	strncpy(lp->ill_tz, inarch.label.ll_tz, PM_TZ_MAXLEN);
}

/*
 * write label records at the start of each physical file
 */
void
writelabel(int do_rewind)
{
    off_t	old_offset;

    if (do_rewind) {
	old_offset = ftell(outarch.logctl.l_tifp);
	rewind(outarch.logctl.l_tifp);
    }
    outarch.logctl.l_label.ill_vol = PM_LOG_VOL_TI;
    __pmLogWriteLabel(outarch.logctl.l_tifp, &outarch.logctl.l_label);
    if (do_rewind)
	fseek(outarch.logctl.l_tifp, (long)old_offset, SEEK_SET);

    if (do_rewind) {
	old_offset = ftell(outarch.logctl.l_mdfp);
	rewind(outarch.logctl.l_mdfp);
    }
    outarch.logctl.l_label.ill_vol = PM_LOG_VOL_META;
    __pmLogWriteLabel(outarch.logctl.l_mdfp, &outarch.logctl.l_label);
    if (do_rewind)
	fseek(outarch.logctl.l_mdfp, (long)old_offset, SEEK_SET);

    if (do_rewind) {
	old_offset = ftell(outarch.logctl.l_mfp);
	rewind(outarch.logctl.l_mfp);
    }
    outarch.logctl.l_label.ill_vol = 0;
    __pmLogWriteLabel(outarch.logctl.l_mfp, &outarch.logctl.l_label);
    if (do_rewind)
	fseek(outarch.logctl.l_mfp, (long)old_offset, SEEK_SET);
}

/*
 * read next metadata record 
 */
static int
nextmeta()
{
    int			sts;
    __pmLogCtl		*lcp;

    lcp = inarch.ctxp->c_archctl->ac_log;
    if ((sts = _pmLogGet(lcp, PM_LOG_VOL_META, &inarch.metarec)) < 0) {
	if (sts != PM_ERR_EOL) {
	    fprintf(stderr, "%s: Error: _pmLogGet[meta %s]: %s\n",
		    pmProgname, inarch.name, pmErrStr(sts));
	    _report(lcp->l_mdfp);
	}
	return -1;
    }

    return ntohl(inarch.metarec[1]);
}


/*
 * read next log record
 *
 * return status is
 * 0		ok
 * 1		ok, but volume switched
 * PM_ERR_EOL	end of file
 * -1		fatal error
 */
static int
nextlog(void)
{
    int			sts;
    __pmLogCtl		*lcp;
    int			old_vol;


    lcp = inarch.ctxp->c_archctl->ac_log;
    old_vol = inarch.ctxp->c_archctl->ac_log->l_curvol;

    if ((sts = __pmLogRead(lcp, PM_MODE_FORW, NULL, &inarch.rp)) < 0) {
	if (sts != PM_ERR_EOL) {
	    fprintf(stderr, "%s: Error: __pmLogRead[log %s]: %s\n",
		    pmProgname, inarch.name, pmErrStr(sts));
	    _report(lcp->l_mfp);
	}
	return -1;
    }

    return old_vol == inarch.ctxp->c_archctl->ac_log->l_curvol ? 0 : 1;
}

#ifdef IS_MINGW
#define SEP '\\'
#else
#define SEP '/'
#endif

/*
 * parse command line arguments
 */
int
parseargs(int argc, char *argv[])
{
    int			c;
    int			sts;
    int			errflag = 0;
    struct stat		sbuf;

    while ((c = getopt(argc, argv, "c:CdD:svw?")) != EOF) {
	switch (c) {

	case 'c':	/* config file */
	    if (stat(optarg, &sbuf) < 0) {
		fprintf(stderr, "%s: stat(%s) failed: %s\n",
			pmProgname, optarg, osstrerror());
		errflag++;
		break;
	    }
#ifndef S_ISLINK
#define S_ISLINK(mode) ((mode & S_IFMT) == S_IFLNK)
#endif
	    if (S_ISREG(sbuf.st_mode) || S_ISLINK(sbuf.st_mode)) {
		nconf++;
		if ((conf = (char **)realloc(conf, nconf*sizeof(conf[0]))) != NULL)
		    conf[nconf-1] = optarg;
	    }
	    else if (S_ISDIR(sbuf.st_mode)) {
		DIR		*dirp;
		struct dirent	*dp;
		char		path[MAXPATHLEN+1];

		if ((dirp = opendir(optarg)) == NULL) {
		    fprintf(stderr, "%s: opendir(%s) failed: %s\n", pmProgname, optarg, osstrerror());
		    errflag++;
		}
		else while ((dp = readdir(dirp)) != NULL) {
		    /* skip ., .. and "hidden" files */
		    if (dp->d_name[0] == '.') continue;
		    snprintf(path, sizeof(path), "%s%c%s", optarg, SEP, dp->d_name);
		    if (stat(path, &sbuf) < 0) {
			fprintf(stderr, "%s: %s: %s\n",
				pmProgname, path, osstrerror());
			errflag++;
		    }
		    if (sbuf.st_mode & (S_IFREG | S_IFLNK)) {
			nconf++;
			if ((conf = (char **)realloc(conf, nconf*sizeof(conf[0]))) == NULL)
			    break;
			if ((conf[nconf-1] = strdup(path)) == NULL) {
			    fprintf(stderr, "conf[%d] strdup(%s) failed: %s\n", nconf-1, path, strerror(errno));
			    exit(1);
			}

		    }
		}
	    }
	    else {
		fprintf(stderr, "Error: -c config %s is not a file or directory\n", optarg);
		errflag++;
	    }
	    if (nconf > 0 && conf == NULL) {
		fprintf(stderr, "conf[%d] realloc(%d) failed: %s\n", nconf, (int)(nconf*sizeof(conf[0])), strerror(errno));
		exit(1);
	    }
	    break;

	case 'C':	/* parse configs and quit */
	    Cflag = 1;
	    vflag = 1;
	    break;

	case 'd':	/* desperate */
	    dflag = 1;
	    break;

	case 'D':	/* debug flag */
	    sts = __pmParseDebug(optarg);
	    if (sts < 0) {
		fprintf(stderr, "%s: unrecognized debug flag specification (%s)\n",
		    pmProgname, optarg);
		errflag++;
	    }
	    else
		pmDebug |= sts;
	    break;

	case 's':	/* do scale conversions */
	    sflag = 1;
	    break;

	case 'v':	/* verbosity */
	    vflag++;
	    break;

	case 'w':	/* print warnings */
	    wflag = 1;
	    break;

	case '?':
	default:
	    errflag++;
	    break;
	}
    }

    if (errflag == 0 && optind != argc-2)
	errflag++;

    return -errflag;
}

static void
parseconfig(char *file)
{
    extern FILE * yyin;

    configfile = file;
    if ((yyin = fopen(configfile, "r")) == NULL) {
	fprintf(stderr, "%s: Cannot open config file \"%s\": %s\n",
		pmProgname, configfile, osstrerror());
	exit(1);
    }
    if (vflag > 1)
	fprintf(stderr, "Start configfile: %s\n", file);
    lineno = 1;

    if (yyparse() != 0)
	exit(1);

    fclose(yyin);
    yyin = NULL;

    return;
}

char *
SemStr(int sem)
{
    static char	buf[20];

    if (sem == PM_SEM_COUNTER) snprintf(buf, sizeof(buf), "counter");
    else if (sem == PM_SEM_INSTANT) snprintf(buf, sizeof(buf), "instant");
    else if (sem == PM_SEM_DISCRETE) snprintf(buf, sizeof(buf), "discrete");
    else snprintf(buf, sizeof(buf), "bad sem? %d", sem);

    return buf;
}

static void
reportconfig(void)
{
    indomspec_t		*ip;
    metricspec_t	*mp;
    int			i;
    int			change = 0;

    printf("PCP Archive Log Rewrite Specifications Summary\n");
    change |= (global.flags != 0);
    if (global.flags & GLOBAL_CHANGE_HOSTNAME)
	printf("Hostname:\t%s -> %s\n", inarch.label.ll_hostname, global.hostname);
    if (global.flags & GLOBAL_CHANGE_TZ)
	printf("Timezone:\t%s -> %s\n", inarch.label.ll_tz, global.tz);
    if (global.flags & GLOBAL_CHANGE_TIME) {
	static struct tm	*tmp;
	char			*sign = "";
	time_t			time;
	if (global.time.tv_sec < 0) {
	    time = (time_t)(-global.time.tv_sec);
	    sign = "-";
	}
	else
	    time = (time_t)global.time.tv_sec;
	tmp = gmtime(&time);
	tmp->tm_hour += 24 * tmp->tm_yday;
	if (tmp->tm_hour < 10)
	    printf("Delta:\t\t-> %s%02d:%02d:%02d.%06d\n", sign, tmp->tm_hour, tmp->tm_min, tmp->tm_sec, (int)global.time.tv_usec);
	else
	    printf("Delta:\t\t-> %s%d:%02d:%02d.%06d\n", sign, tmp->tm_hour, tmp->tm_min, tmp->tm_sec, (int)global.time.tv_usec);
    }
    for (ip = indom_root; ip != NULL; ip = ip->i_next) {
	int		hdr_done = 0;
	if (ip->new_indom != ip->old_indom) {
	    printf("\nInstance Domain: %s\n", pmInDomStr(ip->old_indom));
	    hdr_done = 1;
	    printf("pmInDom:\t-> %s\n", pmInDomStr(ip->new_indom));
	    change |= 1;
	}
	for (i = 0; i < ip->numinst; i++) {
	    change |= (ip->flags[i] != 0);
	    if (ip->flags[i]) {
		if (hdr_done == 0) {
		    printf("\nInstance Domain: %s\n", pmInDomStr(ip->old_indom));
		    hdr_done = 1;
		}
		printf("Instance:\t\[%d] \"%s\" -> ", ip->old_inst[i], ip->old_iname[i]);
		if (ip->flags[i] & INST_DELETE)
		    printf("DELETE\n");
		else {
		    if (ip->flags[i] & INST_CHANGE_INST)
			printf("[%d] ", ip->new_inst[i]);
		    else
			printf("[%d] ", ip->old_inst[i]);
		    if (ip->flags[i] & INST_CHANGE_INAME)
			printf("\"%s\"\n", ip->new_iname[i]);
		    else
			printf("\"%s\"\n", ip->old_iname[i]);
		}
	    }
	}
    }
    for (mp = metric_root; mp != NULL; mp = mp->m_next) {
	if (mp->flags != 0 || mp->ip != NULL) {
	    change |= 1;
	    printf("\nMetric: %s (%s)\n", mp->old_name, pmIDStr(mp->old_desc.pmid));
	}
	if (mp->flags & METRIC_CHANGE_PMID) {
	    printf("pmID:\t\t%s ->", pmIDStr(mp->old_desc.pmid));
	    printf(" %s\n", pmIDStr(mp->new_desc.pmid));
	}
	if (mp->flags & METRIC_CHANGE_NAME)
	    printf("Name:\t\t%s -> %s\n", mp->old_name, mp->new_name);
	if (mp->flags & METRIC_CHANGE_TYPE) {
	    printf("Type:\t\t%s ->", pmTypeStr(mp->old_desc.type));
	    printf(" %s\n", pmTypeStr(mp->new_desc.type));
	}
	if (mp->flags & METRIC_CHANGE_INDOM) {
	    printf("InDom:\t\t%s ->", pmInDomStr(mp->old_desc.indom));
	    printf(" %s\n", pmInDomStr(mp->new_desc.indom));
	    if (mp->output != OUTPUT_ALL) {
		printf("Output:\t\t");
		switch (mp->output) {
		    case OUTPUT_ONE:
			if (mp->old_desc.indom != PM_INDOM_NULL) {
			    printf("value for instance");
			    if (mp->one_inst != PM_IN_NULL)
				printf(" %d", mp->one_inst);
			    if (mp->one_name != NULL)
				printf(" \"%s\"", mp->one_name);
			    putchar('\n');
			}
			else
			    printf("the only value (output instance %d)\n", mp->one_inst);
			break;
		    case OUTPUT_FIRST:
			if (mp->old_desc.indom != PM_INDOM_NULL)
			    printf("first value\n");
			else {
			    if (mp->one_inst != PM_IN_NULL)
				printf("first and only value (output instance %d)\n", mp->one_inst);
			    else
				printf("first and only value (output instance \"%s\")\n", mp->one_name);
			}
			break;
		    case OUTPUT_LAST:
			if (mp->old_desc.indom != PM_INDOM_NULL)
			    printf("last value\n");
			else
			    printf("last and only value (output instance %d)\n", mp->one_inst);
			break;
		    case OUTPUT_MIN:
			if (mp->old_desc.indom != PM_INDOM_NULL)
			    printf("smallest value\n");
			else
			    printf("smallest and only value (output instance %d)\n", mp->one_inst);
			break;
		    case OUTPUT_MAX:
			if (mp->old_desc.indom != PM_INDOM_NULL)
			    printf("largest value\n");
			else
			    printf("largest and only value (output instance %d)\n", mp->one_inst);
			break;
		    case OUTPUT_SUM:
			if (mp->old_desc.indom != PM_INDOM_NULL)
			    printf("sum value (output instance %d)\n", mp->one_inst);
			else
			    printf("sum and only value (output instance %d)\n", mp->one_inst);
			break;
		    case OUTPUT_AVG:
			if (mp->old_desc.indom != PM_INDOM_NULL)
			    printf("average value (output instance %d)\n", mp->one_inst);
			else
			    printf("average and only value (output instance %d)\n", mp->one_inst);
			break;
		}
	    }
	}
	if (mp->ip != NULL)
	    printf("Inst Changes:\t<- InDom %s", pmInDomStr(mp->ip->old_indom));
	if (mp->flags & METRIC_CHANGE_SEM) {
	    printf("Semantics:\t%s ->", SemStr(mp->old_desc.sem));
	    printf(" %s\n", SemStr(mp->new_desc.sem));
	}
	if (mp->flags & METRIC_CHANGE_UNITS) {
	    printf("Units:\t\t%s ->", pmUnitsStr(&mp->old_desc.units));
	    printf(" %s\n", pmUnitsStr(&mp->new_desc.units));
	}
	if (mp->flags & METRIC_DELETE)
	    printf("DELETE\n");
    }
    if (change == 0)
	printf("No changes\n");
}

static int
fixstamp(struct timeval *tvp)
{
    if (global.flags & GLOBAL_CHANGE_TIME) {
	if (global.time.tv_sec > 0) {
	    tvp->tv_sec += global.time.tv_sec;
	    tvp->tv_usec += global.time.tv_usec;
	    if (tvp->tv_usec > 1000000) {
		tvp->tv_sec++;
		tvp->tv_usec -= 1000000;
	    }
	    return 1;
	}
	else if (global.time.tv_sec < 0) {
	    /* parser makes tv_sec < 0 and tv_usec >= 0 */
	    tvp->tv_sec += global.time.tv_sec;
	    tvp->tv_usec -= global.time.tv_usec;
	    if (tvp->tv_usec < 0) {
		tvp->tv_sec--;
		tvp->tv_usec += 1000000;
	    }
	    return 1;
	}
    }
    return 0;
}

/*
 * Link metricspec_t entries to corresponding indom_t entry if there
 * are changes to instance identifiers or instance names (includes
 * instance deletion)
 */
static void
link_entries(void)
{
    indomspec_t		*ip;
    metricspec_t	*mp;
    __pmHashCtl		*hcp;
    __pmHashNode	*this;
    int			i;
    int			change;

    hcp = &inarch.ctxp->c_archctl->ac_log->l_hashpmid;
    for (ip = indom_root; ip != NULL; ip = ip->i_next) {
	change = 0;
	for (i = 0; i < ip->numinst; i++)
	    change |= (ip->flags[i] != 0);
	if (change == 0 && ip->new_indom == ip->old_indom)
	    continue;

	for (this = __pmHashWalk(hcp, W_START); this != NULL; this = __pmHashWalk(hcp, W_NEXT)) {
	    mp = start_metric((pmID)(this->key));
	    if (mp->old_desc.indom == ip->old_indom) {
		if (change)
		    mp->ip = ip;
		if (ip->new_indom != ip->old_indom) {
		    if (mp->flags & METRIC_CHANGE_INDOM) {
			/* indom already changed via metric clause */
			if (mp->new_desc.indom != ip->new_indom) {
			    char	strbuf[80];
			    snprintf(strbuf, sizeof(strbuf), "%s", pmInDomStr(mp->new_desc.indom));
			    snprintf(mess, sizeof(mess), "Conflicting indom change for metric %s (%s from metric clause, %s from indom clause)", mp->old_name, strbuf, pmInDomStr(ip->new_indom));
			    yysemantic(mess);
			}
		    }
		    else {
			mp->flags |= METRIC_CHANGE_INDOM;
			mp->new_desc.indom = ip->new_indom;
		    }
		}
	    }
	}
    }
}

static void
check_indoms()
{
    /*
     * For each metric, make sure the output instance domain will be in
     * the output archive.
     * Called after link_entries(), so if an input metric is associated
     * with an instance domain that has any instance rewriting, we're OK.
     * The case to be checked here is a rewritten metric with an indom
     * clause and no associated indomspec_t (so no instance domain changes,
     * but the new indom may not match any indom in the archive.
     */
    metricspec_t	*mp;
    indomspec_t		*ip;
    __pmHashCtl		*hcp;
    __pmHashNode	*this;


    hcp = &inarch.ctxp->c_archctl->ac_log->l_hashindom;

    for (mp = metric_root; mp != NULL; mp = mp->m_next) {
	if (mp->ip != NULL)
	    /* associated indom has instance changes, we're OK */
	    continue;
	if ((mp->flags & METRIC_CHANGE_INDOM) && mp->new_desc.indom != PM_INDOM_NULL) {
	    for (this = __pmHashWalk(hcp, W_START); this != NULL; this = __pmHashWalk(hcp, W_NEXT)) {
		/*
		 * if this indom has an indomspec_t, check that, else
		 * this indom will go to the archive without change
		 */
		for (ip = indom_root; ip != NULL; ip = ip->i_next) {
		    if (ip->old_indom == mp->old_desc.indom)
			break;
		}
		if (ip == NULL) {
		    if ((pmInDom)(this->key) == mp->new_desc.indom)
			/* we're OK */
			break;
		}
		else {
		    if (ip->new_indom != ip->old_indom &&
		        ip->new_indom == mp->new_desc.indom)
			/* we're OK */
			break;
		}
	    }
	    if (this == NULL) {
		snprintf(mess, sizeof(mess), "New indom (%s) for metric %s is not in the output archive", pmInDomStr(mp->new_desc.indom), mp->old_name);
		yysemantic(mess);
	    }
	}
    }

    /*
     * For each modified instance domain, make sure instances are
     * still unique and instance names are unique to the first
     * space.
     */
    for (ip = indom_root; ip != NULL; ip = ip->i_next) {
	int	i;
	for (i = 0; i < ip->numinst; i++) {
	    int		insti;
	    char	*namei;
	    int		j;
	    if (ip->flags[i] & INST_CHANGE_INST)
		insti = ip->new_inst[i];
	    else
		insti = ip->old_inst[i];
	    if (ip->flags[i] & INST_CHANGE_INAME)
		namei = ip->new_iname[i];
	    else
		namei = ip->old_iname[i];
	    for (j = 0; j < ip->numinst; j++) {
		int	instj;
		char	*namej;
		if (i == j)
		    continue;
		if (ip->flags[j] & INST_CHANGE_INST)
		    instj = ip->new_inst[j];
		else
		    instj = ip->old_inst[j];
		if (ip->flags[j] & INST_CHANGE_INAME)
		    namej = ip->new_iname[j];
		else
		    namej = ip->old_iname[j];
		if (insti == instj) {
		    snprintf(mess, sizeof(mess), "Duplicate instance id %d (\"%s\" and \"%s\") for indom %s", insti, namei, namej, pmInDomStr(ip->old_indom));
		    yysemantic(mess);
		}
		if (inst_name_eq(namei, namej) > 0) {
		    snprintf(mess, sizeof(mess), "Duplicate instance name \"%s\" (%d) and \"%s\" (%d) for indom %s", namei, insti, namej, instj, pmInDomStr(ip->old_indom));
		    yysemantic(mess);
		}
	    }
	}
    }
}

static void
check_output()
{
    /*
     * For each metric, if there is an INDOM clause, perform some
     * additional semantic checks and perhaps a name -> instance id
     * mapping.
     *
     * Note instance renumbering happens _after_ value selction from
     * 		the INDOM -> ,,,, OUTPUT clause, so all references to
     * 		instance names and instance ids are relative to the
     * 		"old" set.
     */
    metricspec_t	*mp;
    indomspec_t		*ip;
    __pmHashCtl		*hcp;

    hcp = &inarch.ctxp->c_archctl->ac_log->l_hashindom;

    for (mp = metric_root; mp != NULL; mp = mp->m_next) {
	if ((mp->flags & METRIC_CHANGE_INDOM)) {
	    if (mp->output == OUTPUT_ONE || mp->output == OUTPUT_FIRST) {
		/*
		 * cases here are
		 * INAME "name"
		 * 	=> one_name == "name" and one_inst == PM_IN_NULL
		 * INST id
		 * 	=> one_name == NULL and one_inst = id
		 */
		if (mp->old_desc.indom != PM_INDOM_NULL && mp->output == OUTPUT_ONE) {
		    /*
		     * old metric is not singular, so one_name and one_inst
		     * are used to pick the value
		     * also map one_name -> one_inst 
		     */
		    int		i;
		    ip = start_indom(mp->old_desc.indom);
		    for (i = 0; i < ip->numinst; i++) {
			if (mp->one_name != NULL) {
			    if (inst_name_eq(ip->old_iname[i], mp->one_name) > 0) {
				mp->one_name = NULL;
				mp->one_inst = ip->old_inst[i];
				break;
			    }
			}
			else if (ip->old_inst[i] == mp->one_inst)
			    break;
		    }<commit_msg>Fixup compilation warning from unused variable in logrewrite.<commit_after>/*
 * pmlogrewrite - config-driven stream editor for PCP archives
 *
 * Copyright (c) 1997-2002 Silicon Graphics, Inc.  All Rights Reserved.
 * Copyright (c) 2011 <NAME>.  All Rights Reserved.
 * 
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the
 * Free Software Foundation; either version 2 of the License, or (at your
 * option) any later version.
 * 
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 * for more details.
 */

#include <math.h>
#include <ctype.h>
#include <sys/stat.h>
#include "pmapi.h"
#include "impl.h"
#include "logger.h"
#include <assert.h>

global_t	global;
indomspec_t	*indom_root = NULL;
metricspec_t	*metric_root = NULL;
int		lineno;

/*
 *  Usage
 */
static void
usage(void)
{
    fprintf(stderr,
"Usage: %s [options] input-archive output-archive\n\
\n\
Options:\n\
  -c config   file or directory to load rules from\n\
  -C          parse config file(s) and quit\n\
  -d          desperate, save output archive even after error\n\
  -s          do scale conversion\n\
  -v          verbose\n\
  -w          emit warnings [default is silence]\n",
	pmProgname);
}

/*
 *  Global variables
 */
static int	first_datarec = 1;		/* first record flag */

off_t		new_log_offset;			/* new log offset */
off_t		new_meta_offset;		/* new meta offset */


/* archive control stuff */
inarch_t		inarch;		/* input archive control */
outarch_t		outarch;	/* output archive control */

/* command line args */
int	nconf = 0;			/* number of config files */
char	**conf = NULL;			/* list of config files */
char	*configfile = NULL;		/* current config file */
int	Cflag = 0;			/* -C parse config and quit */
int	dflag = 0;			/* -d desperate */
int	sflag = 0;			/* -s scale values */
int	vflag = 0;			/* -v verbosity */
int	wflag = 0;			/* -w emit warnings */

/*
 *  report that archive is corrupted
 */
static void
_report(FILE *fp)
{
    off_t	here;
    struct stat	sbuf;

    here = lseek(fileno(fp), 0L, SEEK_CUR);
    fprintf(stderr, "%s: Error occurred at byte offset %ld into a file of",
	    pmProgname, (long)here);
    if (fstat(fileno(fp), &sbuf) < 0)
	fprintf(stderr, ": stat: %s\n", osstrerror());
    else
	fprintf(stderr, " %ld bytes.\n", (long)sbuf.st_size);
    fprintf(stderr, "The last record, and the remainder of this file will not be extracted.\n");
    exit(1);
}

/*
 *  switch output volumes
 */
static void
newvolume(int vol)
{
    FILE		*newfp;

    if ((newfp = __pmLogNewFile(outarch.name, vol)) != NULL) {
	fclose(outarch.logctl.l_mfp);
	outarch.logctl.l_mfp = newfp;
	outarch.logctl.l_label.ill_vol = outarch.logctl.l_curvol = vol;
	__pmLogWriteLabel(outarch.logctl.l_mfp, &outarch.logctl.l_label);
	fflush(outarch.logctl.l_mfp);
    }
    else {
	fprintf(stderr, "%s: __pmLogNewFile(%s,%d) Error: %s\n",
		pmProgname, outarch.name, vol, pmErrStr(-oserror()));
	exit(1);
    }
}

/* construct new archive label */
static void
newlabel(void)
{
    __pmLogLabel	*lp = &outarch.logctl.l_label;

    /* copy magic number, pid, host and timezone */
    lp->ill_magic = inarch.label.ll_magic;
    lp->ill_pid = inarch.label.ll_pid;
    if (global.flags & GLOBAL_CHANGE_HOSTNAME)
	strncpy(lp->ill_hostname, global.hostname, PM_LOG_MAXHOSTLEN);
    else
	strncpy(lp->ill_hostname, inarch.label.ll_hostname, PM_LOG_MAXHOSTLEN);
    if (global.flags & GLOBAL_CHANGE_TZ)
	strncpy(lp->ill_tz, global.tz, PM_TZ_MAXLEN);
    else
	strncpy(lp->ill_tz, inarch.label.ll_tz, PM_TZ_MAXLEN);
}

/*
 * write label records at the start of each physical file
 */
void
writelabel(int do_rewind)
{
    off_t	old_offset;

    if (do_rewind) {
	old_offset = ftell(outarch.logctl.l_tifp);
	rewind(outarch.logctl.l_tifp);
    }
    outarch.logctl.l_label.ill_vol = PM_LOG_VOL_TI;
    __pmLogWriteLabel(outarch.logctl.l_tifp, &outarch.logctl.l_label);
    if (do_rewind)
	fseek(outarch.logctl.l_tifp, (long)old_offset, SEEK_SET);

    if (do_rewind) {
	old_offset = ftell(outarch.logctl.l_mdfp);
	rewind(outarch.logctl.l_mdfp);
    }
    outarch.logctl.l_label.ill_vol = PM_LOG_VOL_META;
    __pmLogWriteLabel(outarch.logctl.l_mdfp, &outarch.logctl.l_label);
    if (do_rewind)
	fseek(outarch.logctl.l_mdfp, (long)old_offset, SEEK_SET);

    if (do_rewind) {
	old_offset = ftell(outarch.logctl.l_mfp);
	rewind(outarch.logctl.l_mfp);
    }
    outarch.logctl.l_label.ill_vol = 0;
    __pmLogWriteLabel(outarch.logctl.l_mfp, &outarch.logctl.l_label);
    if (do_rewind)
	fseek(outarch.logctl.l_mfp, (long)old_offset, SEEK_SET);
}

/*
 * read next metadata record 
 */
static int
nextmeta()
{
    int			sts;
    __pmLogCtl		*lcp;

    lcp = inarch.ctxp->c_archctl->ac_log;
    if ((sts = _pmLogGet(lcp, PM_LOG_VOL_META, &inarch.metarec)) < 0) {
	if (sts != PM_ERR_EOL) {
	    fprintf(stderr, "%s: Error: _pmLogGet[meta %s]: %s\n",
		    pmProgname, inarch.name, pmErrStr(sts));
	    _report(lcp->l_mdfp);
	}
	return -1;
    }

    return ntohl(inarch.metarec[1]);
}


/*
 * read next log record
 *
 * return status is
 * 0		ok
 * 1		ok, but volume switched
 * PM_ERR_EOL	end of file
 * -1		fatal error
 */
static int
nextlog(void)
{
    int			sts;
    __pmLogCtl		*lcp;
    int			old_vol;


    lcp = inarch.ctxp->c_archctl->ac_log;
    old_vol = inarch.ctxp->c_archctl->ac_log->l_curvol;

    if ((sts = __pmLogRead(lcp, PM_MODE_FORW, NULL, &inarch.rp)) < 0) {
	if (sts != PM_ERR_EOL) {
	    fprintf(stderr, "%s: Error: __pmLogRead[log %s]: %s\n",
		    pmProgname, inarch.name, pmErrStr(sts));
	    _report(lcp->l_mfp);
	}
	return -1;
    }

    return old_vol == inarch.ctxp->c_archctl->ac_log->l_curvol ? 0 : 1;
}

#ifdef IS_MINGW
#define SEP '\\'
#else
#define SEP '/'
#endif

/*
 * parse command line arguments
 */
int
parseargs(int argc, char *argv[])
{
    int			c;
    int			sts;
    int			errflag = 0;
    struct stat		sbuf;

    while ((c = getopt(argc, argv, "c:CdD:svw?")) != EOF) {
	switch (c) {

	case 'c':	/* config file */
	    if (stat(optarg, &sbuf) < 0) {
		fprintf(stderr, "%s: stat(%s) failed: %s\n",
			pmProgname, optarg, osstrerror());
		errflag++;
		break;
	    }
#ifndef S_ISLINK
#define S_ISLINK(mode) ((mode & S_IFMT) == S_IFLNK)
#endif
	    if (S_ISREG(sbuf.st_mode) || S_ISLINK(sbuf.st_mode)) {
		nconf++;
		if ((conf = (char **)realloc(conf, nconf*sizeof(conf[0]))) != NULL)
		    conf[nconf-1] = optarg;
	    }
	    else if (S_ISDIR(sbuf.st_mode)) {
		DIR		*dirp;
		struct dirent	*dp;
		char		path[MAXPATHLEN+1];

		if ((dirp = opendir(optarg)) == NULL) {
		    fprintf(stderr, "%s: opendir(%s) failed: %s\n", pmProgname, optarg, osstrerror());
		    errflag++;
		}
		else while ((dp = readdir(dirp)) != NULL) {
		    /* skip ., .. and "hidden" files */
		    if (dp->d_name[0] == '.') continue;
		    snprintf(path, sizeof(path), "%s%c%s", optarg, SEP, dp->d_name);
		    if (stat(path, &sbuf) < 0) {
			fprintf(stderr, "%s: %s: %s\n",
				pmProgname, path, osstrerror());
			errflag++;
		    }
		    if (sbuf.st_mode & (S_IFREG | S_IFLNK)) {
			nconf++;
			if ((conf = (char **)realloc(conf, nconf*sizeof(conf[0]))) == NULL)
			    break;
			if ((conf[nconf-1] = strdup(path)) == NULL) {
			    fprintf(stderr, "conf[%d] strdup(%s) failed: %s\n", nconf-1, path, strerror(errno));
			    exit(1);
			}

		    }
		}
	    }
	    else {
		fprintf(stderr, "Error: -c config %s is not a file or directory\n", optarg);
		errflag++;
	    }
	    if (nconf > 0 && conf == NULL) {
		fprintf(stderr, "conf[%d] realloc(%d) failed: %s\n", nconf, (int)(nconf*sizeof(conf[0])), strerror(errno));
		exit(1);
	    }
	    break;

	case 'C':	/* parse configs and quit */
	    Cflag = 1;
	    vflag = 1;
	    break;

	case 'd':	/* desperate */
	    dflag = 1;
	    break;

	case 'D':	/* debug flag */
	    sts = __pmParseDebug(optarg);
	    if (sts < 0) {
		fprintf(stderr, "%s: unrecognized debug flag specification (%s)\n",
		    pmProgname, optarg);
		errflag++;
	    }
	    else
		pmDebug |= sts;
	    break;

	case 's':	/* do scale conversions */
	    sflag = 1;
	    break;

	case 'v':	/* verbosity */
	    vflag++;
	    break;

	case 'w':	/* print warnings */
	    wflag = 1;
	    break;

	case '?':
	default:
	    errflag++;
	    break;
	}
    }

    if (errflag == 0 && optind != argc-2)
	errflag++;

    return -errflag;
}

static void
parseconfig(char *file)
{
    extern FILE * yyin;

    configfile = file;
    if ((yyin = fopen(configfile, "r")) == NULL) {
	fprintf(stderr, "%s: Cannot open config file \"%s\": %s\n",
		pmProgname, configfile, osstrerror());
	exit(1);
    }
    if (vflag > 1)
	fprintf(stderr, "Start configfile: %s\n", file);
    lineno = 1;

    if (yyparse() != 0)
	exit(1);

    fclose(yyin);
    yyin = NULL;

    return;
}

char *
SemStr(int sem)
{
    static char	buf[20];

    if (sem == PM_SEM_COUNTER) snprintf(buf, sizeof(buf), "counter");
    else if (sem == PM_SEM_INSTANT) snprintf(buf, sizeof(buf), "instant");
    else if (sem == PM_SEM_DISCRETE) snprintf(buf, sizeof(buf), "discrete");
    else snprintf(buf, sizeof(buf), "bad sem? %d", sem);

    return buf;
}

static void
reportconfig(void)
{
    indomspec_t		*ip;
    metricspec_t	*mp;
    int			i;
    int			change = 0;

    printf("PCP Archive Log Rewrite Specifications Summary\n");
    change |= (global.flags != 0);
    if (global.flags & GLOBAL_CHANGE_HOSTNAME)
	printf("Hostname:\t%s -> %s\n", inarch.label.ll_hostname, global.hostname);
    if (global.flags & GLOBAL_CHANGE_TZ)
	printf("Timezone:\t%s -> %s\n", inarch.label.ll_tz, global.tz);
    if (global.flags & GLOBAL_CHANGE_TIME) {
	static struct tm	*tmp;
	char			*sign = "";
	time_t			time;
	if (global.time.tv_sec < 0) {
	    time = (time_t)(-global.time.tv_sec);
	    sign = "-";
	}
	else
	    time = (time_t)global.time.tv_sec;
	tmp = gmtime(&time);
	tmp->tm_hour += 24 * tmp->tm_yday;
	if (tmp->tm_hour < 10)
	    printf("Delta:\t\t-> %s%02d:%02d:%02d.%06d\n", sign, tmp->tm_hour, tmp->tm_min, tmp->tm_sec, (int)global.time.tv_usec);
	else
	    printf("Delta:\t\t-> %s%d:%02d:%02d.%06d\n", sign, tmp->tm_hour, tmp->tm_min, tmp->tm_sec, (int)global.time.tv_usec);
    }
    for (ip = indom_root; ip != NULL; ip = ip->i_next) {
	int		hdr_done = 0;
	if (ip->new_indom != ip->old_indom) {
	    printf("\nInstance Domain: %s\n", pmInDomStr(ip->old_indom));
	    hdr_done = 1;
	    printf("pmInDom:\t-> %s\n", pmInDomStr(ip->new_indom));
	    change |= 1;
	}
	for (i = 0; i < ip->numinst; i++) {
	    change |= (ip->flags[i] != 0);
	    if (ip->flags[i]) {
		if (hdr_done == 0) {
		    printf("\nInstance Domain: %s\n", pmInDomStr(ip->old_indom));
		    hdr_done = 1;
		}
		printf("Instance:\t\[%d] \"%s\" -> ", ip->old_inst[i], ip->old_iname[i]);
		if (ip->flags[i] & INST_DELETE)
		    printf("DELETE\n");
		else {
		    if (ip->flags[i] & INST_CHANGE_INST)
			printf("[%d] ", ip->new_inst[i]);
		    else
			printf("[%d] ", ip->old_inst[i]);
		    if (ip->flags[i] & INST_CHANGE_INAME)
			printf("\"%s\"\n", ip->new_iname[i]);
		    else
			printf("\"%s\"\n", ip->old_iname[i]);
		}
	    }
	}
    }
    for (mp = metric_root; mp != NULL; mp = mp->m_next) {
	if (mp->flags != 0 || mp->ip != NULL) {
	    change |= 1;
	    printf("\nMetric: %s (%s)\n", mp->old_name, pmIDStr(mp->old_desc.pmid));
	}
	if (mp->flags & METRIC_CHANGE_PMID) {
	    printf("pmID:\t\t%s ->", pmIDStr(mp->old_desc.pmid));
	    printf(" %s\n", pmIDStr(mp->new_desc.pmid));
	}
	if (mp->flags & METRIC_CHANGE_NAME)
	    printf("Name:\t\t%s -> %s\n", mp->old_name, mp->new_name);
	if (mp->flags & METRIC_CHANGE_TYPE) {
	    printf("Type:\t\t%s ->", pmTypeStr(mp->old_desc.type));
	    printf(" %s\n", pmTypeStr(mp->new_desc.type));
	}
	if (mp->flags & METRIC_CHANGE_INDOM) {
	    printf("InDom:\t\t%s ->", pmInDomStr(mp->old_desc.indom));
	    printf(" %s\n", pmInDomStr(mp->new_desc.indom));
	    if (mp->output != OUTPUT_ALL) {
		printf("Output:\t\t");
		switch (mp->output) {
		    case OUTPUT_ONE:
			if (mp->old_desc.indom != PM_INDOM_NULL) {
			    printf("value for instance");
			    if (mp->one_inst != PM_IN_NULL)
				printf(" %d", mp->one_inst);
			    if (mp->one_name != NULL)
				printf(" \"%s\"", mp->one_name);
			    putchar('\n');
			}
			else
			    printf("the only value (output instance %d)\n", mp->one_inst);
			break;
		    case OUTPUT_FIRST:
			if (mp->old_desc.indom != PM_INDOM_NULL)
			    printf("first value\n");
			else {
			    if (mp->one_inst != PM_IN_NULL)
				printf("first and only value (output instance %d)\n", mp->one_inst);
			    else
				printf("first and only value (output instance \"%s\")\n", mp->one_name);
			}
			break;
		    case OUTPUT_LAST:
			if (mp->old_desc.indom != PM_INDOM_NULL)
			    printf("last value\n");
			else
			    printf("last and only value (output instance %d)\n", mp->one_inst);
			break;
		    case OUTPUT_MIN:
			if (mp->old_desc.indom != PM_INDOM_NULL)
			    printf("smallest value\n");
			else
			    printf("smallest and only value (output instance %d)\n", mp->one_inst);
			break;
		    case OUTPUT_MAX:
			if (mp->old_desc.indom != PM_INDOM_NULL)
			    printf("largest value\n");
			else
			    printf("largest and only value (output instance %d)\n", mp->one_inst);
			break;
		    case OUTPUT_SUM:
			if (mp->old_desc.indom != PM_INDOM_NULL)
			    printf("sum value (output instance %d)\n", mp->one_inst);
			else
			    printf("sum and only value (output instance %d)\n", mp->one_inst);
			break;
		    case OUTPUT_AVG:
			if (mp->old_desc.indom != PM_INDOM_NULL)
			    printf("average value (output instance %d)\n", mp->one_inst);
			else
			    printf("average and only value (output instance %d)\n", mp->one_inst);
			break;
		}
	    }
	}
	if (mp->ip != NULL)
	    printf("Inst Changes:\t<- InDom %s", pmInDomStr(mp->ip->old_indom));
	if (mp->flags & METRIC_CHANGE_SEM) {
	    printf("Semantics:\t%s ->", SemStr(mp->old_desc.sem));
	    printf(" %s\n", SemStr(mp->new_desc.sem));
	}
	if (mp->flags & METRIC_CHANGE_UNITS) {
	    printf("Units:\t\t%s ->", pmUnitsStr(&mp->old_desc.units));
	    printf(" %s\n", pmUnitsStr(&mp->new_desc.units));
	}
	if (mp->flags & METRIC_DELETE)
	    printf("DELETE\n");
    }
    if (change == 0)
	printf("No changes\n");
}

static int
fixstamp(struct timeval *tvp)
{
    if (global.flags & GLOBAL_CHANGE_TIME) {
	if (global.time.tv_sec > 0) {
	    tvp->tv_sec += global.time.tv_sec;
	    tvp->tv_usec += global.time.tv_usec;
	    if (tvp->tv_usec > 1000000) {
		tvp->tv_sec++;
		tvp->tv_usec -= 1000000;
	    }
	    return 1;
	}
	else if (global.time.tv_sec < 0) {
	    /* parser makes tv_sec < 0 and tv_usec >= 0 */
	    tvp->tv_sec += global.time.tv_sec;
	    tvp->tv_usec -= global.time.tv_usec;
	    if (tvp->tv_usec < 0) {
		tvp->tv_sec--;
		tvp->tv_usec += 1000000;
	    }
	    return 1;
	}
    }
    return 0;
}

/*
 * Link metricspec_t entries to corresponding indom_t entry if there
 * are changes to instance identifiers or instance names (includes
 * instance deletion)
 */
static void
link_entries(void)
{
    indomspec_t		*ip;
    metricspec_t	*mp;
    __pmHashCtl		*hcp;
    __pmHashNode	*this;
    int			i;
    int			change;

    hcp = &inarch.ctxp->c_archctl->ac_log->l_hashpmid;
    for (ip = indom_root; ip != NULL; ip = ip->i_next) {
	change = 0;
	for (i = 0; i < ip->numinst; i++)
	    change |= (ip->flags[i] != 0);
	if (change == 0 && ip->new_indom == ip->old_indom)
	    continue;

	for (this = __pmHashWalk(hcp, W_START); this != NULL; this = __pmHashWalk(hcp, W_NEXT)) {
	    mp = start_metric((pmID)(this->key));
	    if (mp->old_desc.indom == ip->old_indom) {
		if (change)
		    mp->ip = ip;
		if (ip->new_indom != ip->old_indom) {
		    if (mp->flags & METRIC_CHANGE_INDOM) {
			/* indom already changed via metric clause */
			if (mp->new_desc.indom != ip->new_indom) {
			    char	strbuf[80];
			    snprintf(strbuf, sizeof(strbuf), "%s", pmInDomStr(mp->new_desc.indom));
			    snprintf(mess, sizeof(mess), "Conflicting indom change for metric %s (%s from metric clause, %s from indom clause)", mp->old_name, strbuf, pmInDomStr(ip->new_indom));
			    yysemantic(mess);
			}
		    }
		    else {
			mp->flags |= METRIC_CHANGE_INDOM;
			mp->new_desc.indom = ip->new_indom;
		    }
		}
	    }
	}
    }
}

static void
check_indoms()
{
    /*
     * For each metric, make sure the output instance domain will be in
     * the output archive.
     * Called after link_entries(), so if an input metric is associated
     * with an instance domain that has any instance rewriting, we're OK.
     * The case to be checked here is a rewritten metric with an indom
     * clause and no associated indomspec_t (so no instance domain changes,
     * but the new indom may not match any indom in the archive.
     */
    metricspec_t	*mp;
    indomspec_t		*ip;
    __pmHashCtl		*hcp;
    __pmHashNode	*this;


    hcp = &inarch.ctxp->c_archctl->ac_log->l_hashindom;

    for (mp = metric_root; mp != NULL; mp = mp->m_next) {
	if (mp->ip != NULL)
	    /* associated indom has instance changes, we're OK */
	    continue;
	if ((mp->flags & METRIC_CHANGE_INDOM) && mp->new_desc.indom != PM_INDOM_NULL) {
	    for (this = __pmHashWalk(hcp, W_START); this != NULL; this = __pmHashWalk(hcp, W_NEXT)) {
		/*
		 * if this indom has an indomspec_t, check that, else
		 * this indom will go to the archive without change
		 */
		for (ip = indom_root; ip != NULL; ip = ip->i_next) {
		    if (ip->old_indom == mp->old_desc.indom)
			break;
		}
		if (ip == NULL) {
		    if ((pmInDom)(this->key) == mp->new_desc.indom)
			/* we're OK */
			break;
		}
		else {
		    if (ip->new_indom != ip->old_indom &&
		        ip->new_indom == mp->new_desc.indom)
			/* we're OK */
			break;
		}
	    }
	    if (this == NULL) {
		snprintf(mess, sizeof(mess), "New indom (%s) for metric %s is not in the output archive", pmInDomStr(mp->new_desc.indom), mp->old_name);
		yysemantic(mess);
	    }
	}
    }

    /*
     * For each modified instance domain, make sure instances are
     * still unique and instance names are unique to the first
     * space.
     */
    for (ip = indom_root; ip != NULL; ip = ip->i_next) {
	int	i;
	for (i = 0; i < ip->numinst; i++) {
	    int		insti;
	    char	*namei;
	    int		j;
	    if (ip->flags[i] & INST_CHANGE_INST)
		insti = ip->new_inst[i];
	    else
		insti = ip->old_inst[i];
	    if (ip->flags[i] & INST_CHANGE_INAME)
		namei = ip->new_iname[i];
	    else
		namei = ip->old_iname[i];
	    for (j = 0; j < ip->numinst; j++) {
		int	instj;
		char	*namej;
		if (i == j)
		    continue;
		if (ip->flags[j] & INST_CHANGE_INST)
		    instj = ip->new_inst[j];
		else
		    instj = ip->old_inst[j];
		if (ip->flags[j] & INST_CHANGE_INAME)
		    namej = ip->new_iname[j];
		else
		    namej = ip->old_iname[j];
		if (insti == instj) {
		    snprintf(mess, sizeof(mess), "Duplicate instance id %d (\"%s\" and \"%s\") for indom %s", insti, namei, namej, pmInDomStr(ip->old_indom));
		    yysemantic(mess);
		}
		if (inst_name_eq(namei, namej) > 0) {
		    snprintf(mess, sizeof(mess), "Duplicate instance name \"%s\" (%d) and \"%s\" (%d) for indom %s", namei, insti, namej, instj, pmInDomStr(ip->old_indom));
		    yysemantic(mess);
		}
	    }
	}
    }
}

static void
check_output()
{
    /*
     * For each metric, if there is an INDOM clause, perform some
     * additional semantic checks and perhaps a name -> instance id
     * mapping.
     *
     * Note instance renumbering happens _after_ value selction from
     * 		the INDOM -> ,,,, OUTPUT clause, so all references to
     * 		instance names and instance ids are relative to the
     * 		"old" set.
     */
    metricspec_t	*mp;
    indomspec_t		*ip;

    for (mp = metric_root; mp != NULL; mp = mp->m_next) {
	if ((mp->flags & METRIC_CHANGE_INDOM)) {
	    if (mp->output == OUTPUT_ONE || mp->output == OUTPUT_FIRST) {
		/*
		 * cases here are
		 * INAME "name"
		 * 	=> one_name == "name" and one_inst == PM_IN_NULL
		 * INST id
		 * 	=> one_name == NULL and one_inst = id
		 */
		if (mp->old_desc.indom != PM_INDOM_NULL && mp->output == OUTPUT_ONE) {
		    /*
		     * old metric is not singular, so one_name and one_inst
		     * are used to pick the value
		     * also map one_name -> one_inst 
		     */
		    int		i;
		    ip = start_indom(mp->old_desc.indom);
		    for (i = 0; i < ip->numinst; i++) {
			if (mp->one_name != NULL) {
			    if (inst_name_eq(ip->old_iname[i], mp->one_name) > 0) {
				mp->one_name = NULL;
				mp->one_inst = ip->old_inst[i];
				break;
			    }
			}
			else if (ip->old_inst[i] == mp->one_inst)
			    break;
		    }
{'fraction_non_alphanumeric': 0.1624697921701305, 'fraction_numerical': 0.00652489125181247, 'mean_word_length': 2.85369715030732, 'pattern_counts': {'":': 4, '<': 47, '<?xml version=': 0, '>': 428, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2246511', 'n_tokens_mistral': 17580, 'n_tokens_neox': 16253, 'n_words': 3797}