<commit_before>
/*
 * fmt_compile.c -- "compile" format strings for fmt_scan
 *
 * This code is Copyright (c) 2002, by the authors of nmh.  See the
 * COPYRIGHT file in the root directory of the nmh distribution for
 * complete copyright information.
 *
 * This code compiles the format strings (documented in mh-format(5)) into
 * an internal form to be later processed by fmt_scan.c.
 *
 * What happens here is that the format strings are parsed and an array
 * of struct format structures are returned.  Each format structure is
 * a single operation interpreted by the the routines in fmt_scan.c.
 *
 * There is a NOT a one-to-one correspondence between format strings and
 * format instructions; some functions have side effects that can result
 * in multiple instructions being generated.  The exact list of instructions
 * generated by a format string can be seem with the nmh fmtdump utility.
 *
 * A list of format instructions can be found in fmt_compile.h.
 *
 * If you wish to add a new function, you will need to do the following
 * things:
 *
 * - Add a new instruction to the list of instructions in fmt_compile.h.
 *   Note that test instructions (starting with FT_IF_S_NULL) have special
 *   handling, so if you are NOT writing a test function then you need
 *   to insert it into the list before that _and_ bump all of the
 *   following instruction numbers.
 *
 * - Add the function name to the functable[] array below, and write any
 *   special code that your function may require in terms of parsing
 *   (it very well may not need anything).
 *
 * - Add the code in fmt_scan.c to handle your new function.
 *
 * - Add code to fmtdump.c to display your new function.
 *
 * - Document the new function in the mh-format(5) man page.
 *
 */

#include <h/mh.h>
#include <h/addrsbr.h>
#include <h/tws.h>
#include <h/fmt_scan.h>
#include <h/fmt_compile.h>
#include <h/mts.h>

#ifdef HAVE_SYS_TIME_H
# include <sys/time.h>
#endif
#include <time.h>

/*
 * hash table for deciding if a component is "interesting"
 */
struct comp *wantcomp[128];

static struct format *formatvec;	/* array to hold formats */
static struct format *next_fp;		/* next free format slot */
static struct format *fp;		/* current format slot   */
static struct comp *cm;			/* most recent comp ref  */
static struct ftable *ftbl;		/* most recent func ref  */
static int ncomp;
static int infunction;			/* function nesting cnt  */

extern struct mailname fmt_mnull;

/* ftable->type (argument type) */
#define	TF_COMP    0  	    /* component expected                 */
#define	TF_NUM     1  	    /* number expected                    */
#define	TF_STR     2  	    /* string expected                    */
#define	TF_EXPR    3  	    /* component or func. expected        */
#define	TF_NONE    4  	    /* no argument                        */
#define	TF_MYBOX   5 	    /* special - get current user's mbox  */
#define	TF_NOW     6  	    /* special - get current unix time    */
#define	TF_EXPR_SV 7	    /* like expr but save current str reg */
#define	TF_NOP     8	    /* like expr but no result            */
#define TF_MYNAME  9        /* special - get current name of user */
#define TF_MYHOST  10       /* special - get "local" hostname     */
#define TF_LMBOX   11       /* special - get full local mailbox   */

/* ftable->flags */
/* NB that TFL_PUTS is also used to decide whether the test
 * in a "%<(function)..." should be a string or numeric one.
 */
#define	TFL_PUTS   1	    /* implicit putstr if top level */
#define	TFL_PUTN   2	    /* implicit putnum if top level */

/*
 * The functable array maps between the text names of format functions and
 * the format instructions interpreted by the engine in fmt_scan.c.
 *
 * The elements of this structure are as follows:
 *
 * name -   The name of the function as seen in the format string.  This is
 *	    what maps a particular function name into a format instruction.
 * type -   The type of argument this function expects.  Those types are
 *	    listed above (with the TF_ prefix).  This affects what gets
 *	    placed in the format instruction (the f_un union).
 * f_type - The instruction corresponding to this function (from the list
 *	    in fmt_compile.h).
 * extra  - Used by some functions to provide extra data to the compiler.
 *	    Uses include:
 *		- Providing an alternate instruction to combine a load
 *		  and test operation (see do_if()).
 *		- Passed in f_value in the format instruction to provide
 *		  extra information for the engine (see FT_LV_DAT handling
 *		  in fmt_scan.c).
 *		- Provide a hint as to preprocessing that is required for
 *		  this instruction (see do_name()).
 * flags  - See the definitions for TFL_PUTS & TFL_PUTN above.
 */

struct ftable {
    char *name;		/* function name                  */
    char type;		/* argument type                  */
    char f_type; 	/* fmt type                       */
    char extra;		/* arg. type dependent extra info */
    char flags;
};

static struct ftable functable[] = {
     { "nonzero",    TF_EXPR,	FT_V_NE,	FT_IF_V_NE,	0 },
     { "zero",       TF_EXPR,	FT_V_EQ,	FT_IF_V_EQ,	0 },
     { "eq",         TF_NUM,	FT_V_EQ,	FT_IF_V_EQ,	0 },
     { "ne",         TF_NUM,	FT_V_NE,	FT_IF_V_NE,	0 },
     { "gt",         TF_NUM,	FT_V_GT,	FT_IF_V_GT,	0 },
     { "null",       TF_EXPR,	FT_S_NULL,	FT_IF_S_NULL,	0 },
     { "nonnull",    TF_EXPR,	FT_S_NONNULL,	FT_IF_S,	0 },
     { "match",      TF_STR,	FT_V_MATCH,	FT_IF_MATCH,	0 },
     { "amatch",     TF_STR,	FT_V_AMATCH,	FT_IF_AMATCH,	0 },

     { "putstr",     TF_EXPR,	FT_STR,		0,		0 },
     { "putstrf",    TF_EXPR,	FT_STRF,	0,		0 },
     { "putnum",     TF_EXPR,	FT_NUM,		0,		0 },
     { "putnumf",    TF_EXPR,	FT_NUMF,	0,		0 },
     { "putaddr",    TF_STR,	FT_PUTADDR,	0,		0 },
     { "putlit",     TF_STR,	FT_STRLIT,	0,		0 },
     { "zputlit",    TF_STR,	FT_STRLITZ,	0,		0 },
     { "void",       TF_NOP,	0,		0,		0 },<commit_msg>Change %(putlit) and %(zputlit) to take TF_EXPR instead of TF_STR.  This was originally my fault; TF_STR means a string literal as an argument, but the documentation (and the code) really means for it to take an expression.<commit_after>
/*
 * fmt_compile.c -- "compile" format strings for fmt_scan
 *
 * This code is Copyright (c) 2002, by the authors of nmh.  See the
 * COPYRIGHT file in the root directory of the nmh distribution for
 * complete copyright information.
 *
 * This code compiles the format strings (documented in mh-format(5)) into
 * an internal form to be later processed by fmt_scan.c.
 *
 * What happens here is that the format strings are parsed and an array
 * of struct format structures are returned.  Each format structure is
 * a single operation interpreted by the the routines in fmt_scan.c.
 *
 * There is a NOT a one-to-one correspondence between format strings and
 * format instructions; some functions have side effects that can result
 * in multiple instructions being generated.  The exact list of instructions
 * generated by a format string can be seem with the nmh fmtdump utility.
 *
 * A list of format instructions can be found in fmt_compile.h.
 *
 * If you wish to add a new function, you will need to do the following
 * things:
 *
 * - Add a new instruction to the list of instructions in fmt_compile.h.
 *   Note that test instructions (starting with FT_IF_S_NULL) have special
 *   handling, so if you are NOT writing a test function then you need
 *   to insert it into the list before that _and_ bump all of the
 *   following instruction numbers.
 *
 * - Add the function name to the functable[] array below, and write any
 *   special code that your function may require in terms of parsing
 *   (it very well may not need anything).
 *
 * - Add the code in fmt_scan.c to handle your new function.
 *
 * - Add code to fmtdump.c to display your new function.
 *
 * - Document the new function in the mh-format(5) man page.
 *
 */

#include <h/mh.h>
#include <h/addrsbr.h>
#include <h/tws.h>
#include <h/fmt_scan.h>
#include <h/fmt_compile.h>
#include <h/mts.h>

#ifdef HAVE_SYS_TIME_H
# include <sys/time.h>
#endif
#include <time.h>

/*
 * hash table for deciding if a component is "interesting"
 */
struct comp *wantcomp[128];

static struct format *formatvec;	/* array to hold formats */
static struct format *next_fp;		/* next free format slot */
static struct format *fp;		/* current format slot   */
static struct comp *cm;			/* most recent comp ref  */
static struct ftable *ftbl;		/* most recent func ref  */
static int ncomp;
static int infunction;			/* function nesting cnt  */

extern struct mailname fmt_mnull;

/* ftable->type (argument type) */
#define	TF_COMP    0  	    /* component expected                 */
#define	TF_NUM     1  	    /* number expected                    */
#define	TF_STR     2  	    /* string expected                    */
#define	TF_EXPR    3  	    /* component or func. expected        */
#define	TF_NONE    4  	    /* no argument                        */
#define	TF_MYBOX   5 	    /* special - get current user's mbox  */
#define	TF_NOW     6  	    /* special - get current unix time    */
#define	TF_EXPR_SV 7	    /* like expr but save current str reg */
#define	TF_NOP     8	    /* like expr but no result            */
#define TF_MYNAME  9        /* special - get current name of user */
#define TF_MYHOST  10       /* special - get "local" hostname     */
#define TF_LMBOX   11       /* special - get full local mailbox   */

/* ftable->flags */
/* NB that TFL_PUTS is also used to decide whether the test
 * in a "%<(function)..." should be a string or numeric one.
 */
#define	TFL_PUTS   1	    /* implicit putstr if top level */
#define	TFL_PUTN   2	    /* implicit putnum if top level */

/*
 * The functable array maps between the text names of format functions and
 * the format instructions interpreted by the engine in fmt_scan.c.
 *
 * The elements of this structure are as follows:
 *
 * name -   The name of the function as seen in the format string.  This is
 *	    what maps a particular function name into a format instruction.
 * type -   The type of argument this function expects.  Those types are
 *	    listed above (with the TF_ prefix).  This affects what gets
 *	    placed in the format instruction (the f_un union).
 * f_type - The instruction corresponding to this function (from the list
 *	    in fmt_compile.h).
 * extra  - Used by some functions to provide extra data to the compiler.
 *	    Uses include:
 *		- Providing an alternate instruction to combine a load
 *		  and test operation (see do_if()).
 *		- Passed in f_value in the format instruction to provide
 *		  extra information for the engine (see FT_LV_DAT handling
 *		  in fmt_scan.c).
 *		- Provide a hint as to preprocessing that is required for
 *		  this instruction (see do_name()).
 * flags  - See the definitions for TFL_PUTS & TFL_PUTN above.
 */

struct ftable {
    char *name;		/* function name                  */
    char type;		/* argument type                  */
    char f_type; 	/* fmt type                       */
    char extra;		/* arg. type dependent extra info */
    char flags;
};

static struct ftable functable[] = {
     { "nonzero",    TF_EXPR,	FT_V_NE,	FT_IF_V_NE,	0 },
     { "zero",       TF_EXPR,	FT_V_EQ,	FT_IF_V_EQ,	0 },
     { "eq",         TF_NUM,	FT_V_EQ,	FT_IF_V_EQ,	0 },
     { "ne",         TF_NUM,	FT_V_NE,	FT_IF_V_NE,	0 },
     { "gt",         TF_NUM,	FT_V_GT,	FT_IF_V_GT,	0 },
     { "null",       TF_EXPR,	FT_S_NULL,	FT_IF_S_NULL,	0 },
     { "nonnull",    TF_EXPR,	FT_S_NONNULL,	FT_IF_S,	0 },
     { "match",      TF_STR,	FT_V_MATCH,	FT_IF_MATCH,	0 },
     { "amatch",     TF_STR,	FT_V_AMATCH,	FT_IF_AMATCH,	0 },

     { "putstr",     TF_EXPR,	FT_STR,		0,		0 },
     { "putstrf",    TF_EXPR,	FT_STRF,	0,		0 },
     { "putnum",     TF_EXPR,	FT_NUM,		0,		0 },
     { "putnumf",    TF_EXPR,	FT_NUMF,	0,		0 },
     { "putaddr",    TF_STR,	FT_PUTADDR,	0,		0 },
     { "putlit",     TF_EXPR,	FT_STRLIT,	0,		0 },
     { "zputlit",    TF_EXPR,	FT_STRLITZ,	0,		0 },
     { "void",       TF_NOP,	0,		0,		0 },
{'fraction_non_alphanumeric': 0.09184350132625994, 'fraction_numerical': 0.008454907161803714, 'mean_word_length': 2.5236565420560746, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6465429', 'n_tokens_mistral': 3959, 'n_tokens_neox': 3880, 'n_words': 1528}