<commit_before>
/*
 * fmt_scan.c -- format string interpretation
 *
 * $Id$
 *
 * This code is Copyright (c) 2002, by the authors of nmh.  See the
 * COPYRIGHT file in the root directory of the nmh distribution for
 * complete copyright information.
 */

#include <h/mh.h>
#include <h/addrsbr.h>
#include <h/fmt_scan.h>
#include <h/tws.h>
#include <h/fmt_compile.h>

#ifdef TIME_WITH_SYS_TIME
# include <sys/time.h>
# include <time.h>
#else
# ifdef TM_IN_SYS_TIME
#  include <sys/time.h>
# else
#  include <time.h>
# endif
#endif

#define	NFMTS MAXARGS

extern char *formataddr ();	/* hook for custom address formatting */

#ifdef LBL
struct msgs *fmt_current_folder; /* current folder (set by main program) */
#endif

extern int fmt_norm;		/* defined in sbr/fmt_def.c = AD_NAME */
struct mailname fmt_mnull;

/*
 * static prototypes
 */
static int match (char *, char *);
static char *get_x400_friendly (char *, char *, int);
static int get_x400_comp (char *, char *, char *, int);


/*
 * test if string "sub" appears anywhere in
 * string "str" (case insensitive).
 */

static int
match (char *str, char *sub)
{
    int c1, c2;
    char *s1, *s2;

#ifdef LOCALE
    while ((c1 = *sub)) {
	c1 = (isalpha(c1) && isupper(c1)) ? tolower(c1) : c1;
	while ((c2 = *str++) && c1 != ((isalpha(c2) && isupper(c2)) ? tolower(c2) : c2))
	    ;
	if (! c2)
	    return 0;
	s1 = sub + 1; s2 = str;
	while ((c1 = *s1++) && ((isalpha(c1) && isupper(c1)) ? tolower(c1) : c1) == ((isalpha(c2 =*s2++) && isupper(c2)) ? tolower(c2) : c2))
	    ;
	if (! c1)
	    return 1;
    }
#else
    while ((c1 = *sub)) {
	while ((c2 = *str++) && (c1 | 040) != (c2 | 040))
	    ;
	if (! c2)
	    return 0;
	s1 = sub + 1; s2 = str;
	while ((c1 = *s1++) && (c1 | 040) == (*s2++ | 040))
	    ;
	if (! c1)
	    return 1;
    }
#endif
    return 1;
}

/*
 * macros to format data
 */

#define PUTDF(cp, num, wid, fill)\
	if (cp + wid < ep) {\
	    if ((i = (num)) < 0)\
		i = -(num);\
	    if ((c = (wid)) < 0)\
		c = -c;\
	    sp = cp + c;\
	    do {\
		*--sp = (i % 10) + '0';\
		i /= 10;\
	    } while (i > 0 && sp > cp);\
	    if (i > 0)\
		*sp = '?';\
	    else if ((num) < 0 && sp > cp)\
		*--sp = '-';\
	    while (sp > cp)\
		*--sp = fill;\
	    cp += c;\
	    }

#define PUTD(cp, num)\
	if (cp < ep) {\
	    if ((i = (num)) == 0)\
		*cp++ = '0';\
	    else {\
		if ((i = (num)) < 0) {\
		    *cp++ = '-';\
		    i = -(num);\
		    }\
		c = 10;\
		while (c <= i) \
		    c *= 10;\
		while (cp < ep && c > 1) {\
		    c /= 10;\
		    *cp++ = (i / c) + '0';\
		    i %= c;\
		    }\
		}\
	    }

#ifdef LOCALE
#define PUTSF(cp, str, wid, fill) {\
		ljust = 0;\
		if ((i = (wid)) < 0) {\
			i = -i;\
			ljust++;\
		}\
		if ((sp = (str))) {\
			if (ljust) {\
				c = strlen(sp);\
				if (c > i)\
					sp += c - i;\
				else {\
					while( --i >= c && cp < ep)\
						*cp++ = fill;\
					i++;\
				}\
			} else {\
			    while ((c = (unsigned char) *sp) && (iscntrl(c) || isspace(c)))\
				sp++;\
			}\
			while ((c = (unsigned char) *sp++) && --i >= 0 && cp < ep)\
				if (isgraph(c)) \
				    *cp++ = c;\
				else {\
					while ((c = (unsigned char) *sp) && (iscntrl(c) || isspace(c)))\
						sp++;\
					    *cp++ = ' ';\
				}\
		}\
		if (!ljust)\
		while( --i >= 0 && cp < ep)\
		    *cp++ = fill;\
	}

#define PUTS(cp, str) {\
		if ((sp = (str))) {\
		    while ((c = (unsigned char) *sp) && (iscntrl(c) || isspace(c)))\
			sp++;\
		    while((c = (unsigned char) *sp++) && cp < ep)\
			if (isgraph(c)) \
			    *cp++ = c;\
			else {\
			    while ((c = (unsigned char) *sp) && (iscntrl(c) || isspace(c)))\
				sp++;\
			    *cp++ = ' ';\
			}\
		}\
	}

#else /* LOCALE */
#define PUTSF(cp, str, wid, fill) {\
		ljust = 0;\
		if ((i = (wid)) < 0) {\
			i = -i;\
			ljust++;\
		}\
		if (sp = (str)) {\
			if (ljust) {\
				c = strlen(sp);\
				if (c > i)\
					sp += c - i;\
				else {\
					while( --i >= c && cp < ep)\
						*cp++ = fill;\
					i++;\
				}\
			} else {\
		    while ((c = *sp) && c <= 32)\
			sp++;\
			}\
			while ((c = *sp++) && --i >= 0 && cp < ep)\
				if (c > 32) \
			    *cp++ = c;\
			else {\
					while ((c = *sp) && c <= 32)\
				sp++;\
			    *cp++ = ' ';\
			}\
		}\
		if (!ljust)\
		while( --i >= 0 && cp < ep)\
		    *cp++ = fill;\
		}

#define PUTS(cp, str) {\
		if (sp = (str)) {\
		    while ((c = *sp) && c <= 32)\
			sp++;\
		    while( (c = *sp++) && cp < ep)\
			if ( c > 32 ) \
			    *cp++ = c;\
			else {\
			    while ( (c = *sp) && c <= 32 )\
				sp++;\
			    *cp++ = ' ';\
			}\
		}\
		}

#endif /* LOCALE */


static char *lmonth[] = { "January",  "February","March",   "April",
			  "May",      "June",    "July",    "August",
			  "September","October", "November","December" };

static char *
get_x400_friendly (char *mbox, char *buffer, int buffer_len)
{
    char given[BUFSIZ], surname[BUFSIZ];

    if (mbox == NULL)
	return NULL;
    if (*mbox == '"')
	mbox++;
    if (*mbox != '/')
	return NULL;

    if (get_x400_comp (mbox, "/PN=", buffer, buffer_len)) {
	for (mbox = buffer; (mbox = strchr(mbox, '.')); )
	    *mbox++ = ' ';

	return buffer;
    }

    if (!get_x400_comp (mbox, "/S=", surname, sizeof(surname)))
	return NULL;

    if (get_x400_comp (mbox, "/G=", given, sizeof(given)))
	snprintf (buffer, buffer_len, "%s %s", given, surname);
    else
	snprintf (buffer, buffer_len, "%s", surname);

    return buffer;
}

static int
get_x400_comp (char *mbox, char *key, char *buffer, int buffer_len)
{
    int	idx;
    char *cp;

    if ((idx = stringdex (key, mbox)) < 0
	    || !(cp = strchr(mbox += idx + strlen (key), '/')))
	return 0;

    snprintf (buffer, buffer_len, "%*.*s", cp - mbox, cp - mbox, mbox);
    return 1;
}

struct format *
fmt_scan (struct format *format, char *scanl, int width, int *dat)
{
    char *cp, *ep, *sp;
    char *savestr, *str = NULL;
    char buffer[BUFSIZ], buffer2[BUFSIZ];
    int i, c, ljust;
    int value = 0;
    time_t t;
    struct format *fmt;
    struct comp *comp;
    struct tws *tws;
    struct mailname *mn;

    cp = scanl;
    ep = scanl + width - 1;

    for (fmt = format; fmt->f_type != FT_DONE; fmt++)
	switch (fmt->f_type) {
	case FT_PARSEADDR:
	case FT_PARSEDATE:
	    fmt->f_comp->c_flags &= ~CF_PARSED;
	    break;
	}

    fmt = format;

    while (cp < ep) {
	switch (fmt->f_type) {

	case FT_COMP:
	    PUTS (cp, fmt->f_comp->c_text);
	    break;
	case FT_COMPF:
	    PUTSF (cp, fmt->f_comp->c_text, fmt->f_width, fmt->f_fill);
	    break;

	case FT_LIT:
	    sp = fmt->f_text;
	    while( (c = *sp++) && cp < ep)
		*cp++ = c;
	    break;
	case FT_LITF:
	    sp = fmt->f_text;
	    ljust = 0;
	    i = fmt->f_width;
	    if (i < 0) {
		i = -i;
		ljust++;		/* XXX should do something with this */
	    }
	    while( (c = *sp++) && --i >= 0 && cp < ep)
		*cp++ = c;
	    while( --i >= 0 && cp < ep)
		*cp++ = fmt->f_fill;
	    break;

	case FT_STR:
	    PUTS (cp, str);
	    break;
	case FT_STRF:
	    PUTSF (cp, str, fmt->f_width, fmt->f_fill);
	    break;
	case FT_STRFW:
	    adios (NULL, "internal error (FT_STRFW)");

	case FT_NUM:
	    PUTD (cp, value);
	    break;
	case FT_NUMF:
	    PUTDF (cp, value, fmt->f_width, fmt->f_fill);
	    break;

	case FT_CHAR:
	    *cp++ = fmt->f_char;
	    break;

	case FT_DONE:
	    goto finished;

	case FT_IF_S:
	    if (!(value = (str && *str))) {
		fmt += fmt->f_skip;
		continue;
	    }
	    break;

	case FT_IF_S_NULL:
	    if (!(value = (str == NULL || *str == 0))) {
		fmt += fmt->f_skip;
		continue;
	    }
	    break;

	case FT_IF_V_EQ:
	    if (value != fmt->f_value) {
		fmt += fmt->f_skip;
		continue;
	    }
	    break;

	case FT_IF_V_NE:
	    if (value == fmt->f_value) {
		fmt += fmt->f_skip;
		continue;
	    }
	    break;

	case FT_IF_V_GT:
	    if (value <= fmt->f_value) {
		fmt += fmt->f_skip;
		continue;
	    }
	    break;

	case FT_IF_MATCH:
	    if (!(value = (str && match (str, fmt->f_text)))) {
		fmt += fmt->f_skip;
		continue;
	    }
	    break;

	case FT_V_MATCH:
	    if (str)
		value = match (str, fmt->f_text);
	    else
		value = 0;
	    break;

	case FT_IF_AMATCH:
	    if (!(value = (str && uprf (str, fmt->f_text)))) {
		fmt += fmt->f_skip;
		continue;
	    }
	    break;

	case FT_V_AMATCH:
	    value = uprf (str, fmt->f_text);
	    break;

	case FT_S_NONNULL:
	    value = (str != NULL && *str != 0);
	    break;

	case FT_S_NULL:
	    value = (str == NULL || *str == 0);
	    break;

	case FT_V_EQ:
	    value = (fmt->f_value == value);
	    break;

	case FT_V_NE:
	    value = (fmt->f_value != value);
	    break;

	case FT_V_GT:
	    value = (fmt->f_value > value);
	    break;

	case FT_GOTO:
	    fmt += fmt->f_skip;
	    continue;

	case FT_NOP:
	    break;

	case FT_LS_COMP:
	    str = fmt->f_comp->c_text;
	    break;
	case FT_LS_LIT:
	    str = fmt->f_text;
	    break;
	case FT_LS_GETENV:
	    if (!(str = getenv (fmt->f_text)))
		str = "";
	    break;
	case FT_LS_CFIND:
	    if (!(str = context_find (fmt->f_text)))
		str = "";
	    break;

	case FT_LS_DECODECOMP:
	    if (decode_rfc2047(fmt->f_comp->c_text, buffer2))
		str = buffer2;
	    else
		str = fmt->f_comp->c_text;
	    break;

	case FT_LS_DECODE:
	    if (str && decode_rfc2047(str, buffer2))
		str = buffer2;
	    break;

	case FT_LS_TRIM:
	    if (str) {
		    char *xp;

		    strncpy(buffer, str, sizeof(buffer));
		    str = buffer;
		    while (isspace(*str))
			    str++;
		    ljust = 0;
		    if ((i = fmt->f_width) < 0) {
			    i = -i;
			    ljust++;
		    }

		    if (!ljust && i > 0 && strlen(str) > i)
			    str[i] = '\0';
		    xp = str;
		    xp += strlen(str) - 1;
		    while (xp > str && isspace(*xp))
			    *xp-- = '\0';
		    if (ljust && i > 0 && strlen(str) > i)
			str += strlen(str) - i;
	    }
	    break;

	case FT_LV_COMPFLAG:
	    value = (fmt->f_comp->c_flags & CF_TRUE) != 0;
	    break;
	case FT_LV_COMP:
	    value = (comp = fmt->f_comp)->c_text ? atoi(comp->c_text) : 0;
	    break;
	case FT_LV_LIT:
	    value = fmt->f_value;
	    break;
	case FT_LV_DAT:
	    value = dat[fmt->f_value];
	    break;
	case FT_LV_STRLEN:
	    value = strlen(str);
	    break;
	case FT_LV_CHAR_LEFT:
	    value = width - (cp - scanl);
	    break;
	case FT_LV_PLUS_L:
	    value += fmt->f_value;
	    break;
	case FT_LV_MINUS_L:
	    value = fmt->f_value - value;
	    break;
	case FT_LV_DIVIDE_L:
	    if (fmt->f_value)
		value = value / fmt->f_value;
	    else
		value = 0;
	    break;<commit_msg>[bug #4297] The strlen mh-format function can crash if 'str' is null.<commit_after>
/*
 * fmt_scan.c -- format string interpretation
 *
 * $Id$
 *
 * This code is Copyright (c) 2002, by the authors of nmh.  See the
 * COPYRIGHT file in the root directory of the nmh distribution for
 * complete copyright information.
 */

#include <h/mh.h>
#include <h/addrsbr.h>
#include <h/fmt_scan.h>
#include <h/tws.h>
#include <h/fmt_compile.h>

#ifdef TIME_WITH_SYS_TIME
# include <sys/time.h>
# include <time.h>
#else
# ifdef TM_IN_SYS_TIME
#  include <sys/time.h>
# else
#  include <time.h>
# endif
#endif

#define	NFMTS MAXARGS

extern char *formataddr ();	/* hook for custom address formatting */

#ifdef LBL
struct msgs *fmt_current_folder; /* current folder (set by main program) */
#endif

extern int fmt_norm;		/* defined in sbr/fmt_def.c = AD_NAME */
struct mailname fmt_mnull;

/*
 * static prototypes
 */
static int match (char *, char *);
static char *get_x400_friendly (char *, char *, int);
static int get_x400_comp (char *, char *, char *, int);


/*
 * test if string "sub" appears anywhere in
 * string "str" (case insensitive).
 */

static int
match (char *str, char *sub)
{
    int c1, c2;
    char *s1, *s2;

#ifdef LOCALE
    while ((c1 = *sub)) {
	c1 = (isalpha(c1) && isupper(c1)) ? tolower(c1) : c1;
	while ((c2 = *str++) && c1 != ((isalpha(c2) && isupper(c2)) ? tolower(c2) : c2))
	    ;
	if (! c2)
	    return 0;
	s1 = sub + 1; s2 = str;
	while ((c1 = *s1++) && ((isalpha(c1) && isupper(c1)) ? tolower(c1) : c1) == ((isalpha(c2 =*s2++) && isupper(c2)) ? tolower(c2) : c2))
	    ;
	if (! c1)
	    return 1;
    }
#else
    while ((c1 = *sub)) {
	while ((c2 = *str++) && (c1 | 040) != (c2 | 040))
	    ;
	if (! c2)
	    return 0;
	s1 = sub + 1; s2 = str;
	while ((c1 = *s1++) && (c1 | 040) == (*s2++ | 040))
	    ;
	if (! c1)
	    return 1;
    }
#endif
    return 1;
}

/*
 * macros to format data
 */

#define PUTDF(cp, num, wid, fill)\
	if (cp + wid < ep) {\
	    if ((i = (num)) < 0)\
		i = -(num);\
	    if ((c = (wid)) < 0)\
		c = -c;\
	    sp = cp + c;\
	    do {\
		*--sp = (i % 10) + '0';\
		i /= 10;\
	    } while (i > 0 && sp > cp);\
	    if (i > 0)\
		*sp = '?';\
	    else if ((num) < 0 && sp > cp)\
		*--sp = '-';\
	    while (sp > cp)\
		*--sp = fill;\
	    cp += c;\
	    }

#define PUTD(cp, num)\
	if (cp < ep) {\
	    if ((i = (num)) == 0)\
		*cp++ = '0';\
	    else {\
		if ((i = (num)) < 0) {\
		    *cp++ = '-';\
		    i = -(num);\
		    }\
		c = 10;\
		while (c <= i) \
		    c *= 10;\
		while (cp < ep && c > 1) {\
		    c /= 10;\
		    *cp++ = (i / c) + '0';\
		    i %= c;\
		    }\
		}\
	    }

#ifdef LOCALE
#define PUTSF(cp, str, wid, fill) {\
		ljust = 0;\
		if ((i = (wid)) < 0) {\
			i = -i;\
			ljust++;\
		}\
		if ((sp = (str))) {\
			if (ljust) {\
				c = strlen(sp);\
				if (c > i)\
					sp += c - i;\
				else {\
					while( --i >= c && cp < ep)\
						*cp++ = fill;\
					i++;\
				}\
			} else {\
			    while ((c = (unsigned char) *sp) && (iscntrl(c) || isspace(c)))\
				sp++;\
			}\
			while ((c = (unsigned char) *sp++) && --i >= 0 && cp < ep)\
				if (isgraph(c)) \
				    *cp++ = c;\
				else {\
					while ((c = (unsigned char) *sp) && (iscntrl(c) || isspace(c)))\
						sp++;\
					    *cp++ = ' ';\
				}\
		}\
		if (!ljust)\
		while( --i >= 0 && cp < ep)\
		    *cp++ = fill;\
	}

#define PUTS(cp, str) {\
		if ((sp = (str))) {\
		    while ((c = (unsigned char) *sp) && (iscntrl(c) || isspace(c)))\
			sp++;\
		    while((c = (unsigned char) *sp++) && cp < ep)\
			if (isgraph(c)) \
			    *cp++ = c;\
			else {\
			    while ((c = (unsigned char) *sp) && (iscntrl(c) || isspace(c)))\
				sp++;\
			    *cp++ = ' ';\
			}\
		}\
	}

#else /* LOCALE */
#define PUTSF(cp, str, wid, fill) {\
		ljust = 0;\
		if ((i = (wid)) < 0) {\
			i = -i;\
			ljust++;\
		}\
		if (sp = (str)) {\
			if (ljust) {\
				c = strlen(sp);\
				if (c > i)\
					sp += c - i;\
				else {\
					while( --i >= c && cp < ep)\
						*cp++ = fill;\
					i++;\
				}\
			} else {\
		    while ((c = *sp) && c <= 32)\
			sp++;\
			}\
			while ((c = *sp++) && --i >= 0 && cp < ep)\
				if (c > 32) \
			    *cp++ = c;\
			else {\
					while ((c = *sp) && c <= 32)\
				sp++;\
			    *cp++ = ' ';\
			}\
		}\
		if (!ljust)\
		while( --i >= 0 && cp < ep)\
		    *cp++ = fill;\
		}

#define PUTS(cp, str) {\
		if (sp = (str)) {\
		    while ((c = *sp) && c <= 32)\
			sp++;\
		    while( (c = *sp++) && cp < ep)\
			if ( c > 32 ) \
			    *cp++ = c;\
			else {\
			    while ( (c = *sp) && c <= 32 )\
				sp++;\
			    *cp++ = ' ';\
			}\
		}\
		}

#endif /* LOCALE */


static char *lmonth[] = { "January",  "February","March",   "April",
			  "May",      "June",    "July",    "August",
			  "September","October", "November","December" };

static char *
get_x400_friendly (char *mbox, char *buffer, int buffer_len)
{
    char given[BUFSIZ], surname[BUFSIZ];

    if (mbox == NULL)
	return NULL;
    if (*mbox == '"')
	mbox++;
    if (*mbox != '/')
	return NULL;

    if (get_x400_comp (mbox, "/PN=", buffer, buffer_len)) {
	for (mbox = buffer; (mbox = strchr(mbox, '.')); )
	    *mbox++ = ' ';

	return buffer;
    }

    if (!get_x400_comp (mbox, "/S=", surname, sizeof(surname)))
	return NULL;

    if (get_x400_comp (mbox, "/G=", given, sizeof(given)))
	snprintf (buffer, buffer_len, "%s %s", given, surname);
    else
	snprintf (buffer, buffer_len, "%s", surname);

    return buffer;
}

static int
get_x400_comp (char *mbox, char *key, char *buffer, int buffer_len)
{
    int	idx;
    char *cp;

    if ((idx = stringdex (key, mbox)) < 0
	    || !(cp = strchr(mbox += idx + strlen (key), '/')))
	return 0;

    snprintf (buffer, buffer_len, "%*.*s", cp - mbox, cp - mbox, mbox);
    return 1;
}

struct format *
fmt_scan (struct format *format, char *scanl, int width, int *dat)
{
    char *cp, *ep, *sp;
    char *savestr, *str = NULL;
    char buffer[BUFSIZ], buffer2[BUFSIZ];
    int i, c, ljust;
    int value = 0;
    time_t t;
    struct format *fmt;
    struct comp *comp;
    struct tws *tws;
    struct mailname *mn;

    cp = scanl;
    ep = scanl + width - 1;

    for (fmt = format; fmt->f_type != FT_DONE; fmt++)
	switch (fmt->f_type) {
	case FT_PARSEADDR:
	case FT_PARSEDATE:
	    fmt->f_comp->c_flags &= ~CF_PARSED;
	    break;
	}

    fmt = format;

    while (cp < ep) {
	switch (fmt->f_type) {

	case FT_COMP:
	    PUTS (cp, fmt->f_comp->c_text);
	    break;
	case FT_COMPF:
	    PUTSF (cp, fmt->f_comp->c_text, fmt->f_width, fmt->f_fill);
	    break;

	case FT_LIT:
	    sp = fmt->f_text;
	    while( (c = *sp++) && cp < ep)
		*cp++ = c;
	    break;
	case FT_LITF:
	    sp = fmt->f_text;
	    ljust = 0;
	    i = fmt->f_width;
	    if (i < 0) {
		i = -i;
		ljust++;		/* XXX should do something with this */
	    }
	    while( (c = *sp++) && --i >= 0 && cp < ep)
		*cp++ = c;
	    while( --i >= 0 && cp < ep)
		*cp++ = fmt->f_fill;
	    break;

	case FT_STR:
	    PUTS (cp, str);
	    break;
	case FT_STRF:
	    PUTSF (cp, str, fmt->f_width, fmt->f_fill);
	    break;
	case FT_STRFW:
	    adios (NULL, "internal error (FT_STRFW)");

	case FT_NUM:
	    PUTD (cp, value);
	    break;
	case FT_NUMF:
	    PUTDF (cp, value, fmt->f_width, fmt->f_fill);
	    break;

	case FT_CHAR:
	    *cp++ = fmt->f_char;
	    break;

	case FT_DONE:
	    goto finished;

	case FT_IF_S:
	    if (!(value = (str && *str))) {
		fmt += fmt->f_skip;
		continue;
	    }
	    break;

	case FT_IF_S_NULL:
	    if (!(value = (str == NULL || *str == 0))) {
		fmt += fmt->f_skip;
		continue;
	    }
	    break;

	case FT_IF_V_EQ:
	    if (value != fmt->f_value) {
		fmt += fmt->f_skip;
		continue;
	    }
	    break;

	case FT_IF_V_NE:
	    if (value == fmt->f_value) {
		fmt += fmt->f_skip;
		continue;
	    }
	    break;

	case FT_IF_V_GT:
	    if (value <= fmt->f_value) {
		fmt += fmt->f_skip;
		continue;
	    }
	    break;

	case FT_IF_MATCH:
	    if (!(value = (str && match (str, fmt->f_text)))) {
		fmt += fmt->f_skip;
		continue;
	    }
	    break;

	case FT_V_MATCH:
	    if (str)
		value = match (str, fmt->f_text);
	    else
		value = 0;
	    break;

	case FT_IF_AMATCH:
	    if (!(value = (str && uprf (str, fmt->f_text)))) {
		fmt += fmt->f_skip;
		continue;
	    }
	    break;

	case FT_V_AMATCH:
	    value = uprf (str, fmt->f_text);
	    break;

	case FT_S_NONNULL:
	    value = (str != NULL && *str != 0);
	    break;

	case FT_S_NULL:
	    value = (str == NULL || *str == 0);
	    break;

	case FT_V_EQ:
	    value = (fmt->f_value == value);
	    break;

	case FT_V_NE:
	    value = (fmt->f_value != value);
	    break;

	case FT_V_GT:
	    value = (fmt->f_value > value);
	    break;

	case FT_GOTO:
	    fmt += fmt->f_skip;
	    continue;

	case FT_NOP:
	    break;

	case FT_LS_COMP:
	    str = fmt->f_comp->c_text;
	    break;
	case FT_LS_LIT:
	    str = fmt->f_text;
	    break;
	case FT_LS_GETENV:
	    if (!(str = getenv (fmt->f_text)))
		str = "";
	    break;
	case FT_LS_CFIND:
	    if (!(str = context_find (fmt->f_text)))
		str = "";
	    break;

	case FT_LS_DECODECOMP:
	    if (decode_rfc2047(fmt->f_comp->c_text, buffer2))
		str = buffer2;
	    else
		str = fmt->f_comp->c_text;
	    break;

	case FT_LS_DECODE:
	    if (str && decode_rfc2047(str, buffer2))
		str = buffer2;
	    break;

	case FT_LS_TRIM:
	    if (str) {
		    char *xp;

		    strncpy(buffer, str, sizeof(buffer));
		    str = buffer;
		    while (isspace(*str))
			    str++;
		    ljust = 0;
		    if ((i = fmt->f_width) < 0) {
			    i = -i;
			    ljust++;
		    }

		    if (!ljust && i > 0 && strlen(str) > i)
			    str[i] = '\0';
		    xp = str;
		    xp += strlen(str) - 1;
		    while (xp > str && isspace(*xp))
			    *xp-- = '\0';
		    if (ljust && i > 0 && strlen(str) > i)
			str += strlen(str) - i;
	    }
	    break;

	case FT_LV_COMPFLAG:
	    value = (fmt->f_comp->c_flags & CF_TRUE) != 0;
	    break;
	case FT_LV_COMP:
	    value = (comp = fmt->f_comp)->c_text ? atoi(comp->c_text) : 0;
	    break;
	case FT_LV_LIT:
	    value = fmt->f_value;
	    break;
	case FT_LV_DAT:
	    value = dat[fmt->f_value];
	    break;
	case FT_LV_STRLEN:
	    if (str != NULL)
		    value = strlen(str);
	    else
		    value = 0;
	    break;
	case FT_LV_CHAR_LEFT:
	    value = width - (cp - scanl);
	    break;
	case FT_LV_PLUS_L:
	    value += fmt->f_value;
	    break;
	case FT_LV_MINUS_L:
	    value = fmt->f_value - value;
	    break;
	case FT_LV_DIVIDE_L:
	    if (fmt->f_value)
		value = value / fmt->f_value;
	    else
		value = 0;
	    break;
{'fraction_non_alphanumeric': 0.20108382888931517, 'fraction_numerical': 0.015969691156723576, 'mean_word_length': 2.0173636232093766, 'pattern_counts': {'":': 0, '<': 81, '<?xml version=': 0, '>': 187, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2795367', 'n_tokens_mistral': 9810, 'n_tokens_neox': 8521, 'n_words': 2252}