<commit_before>/*
**++
**  FACILITY:	MMK
**
**  ABSTRACT:	Symbol substitution routines
**
**  MODULE DESCRIPTION:
**
**  	This module contains routines that deal with the
**  MMK global and local symbol lists.
**
**  AUTHOR: 	    <NAME>
**
**  Copyright (c) 2008, <NAME>.
**  Copyright (c) 2012, Endless Software Solutions.
**  
**  All rights reserved.
**  
**  Redistribution and use in source and binary forms, with or without
**  modification, are permitted provided that the following conditions
**  are met:
**  
**      * Redistributions of source code must retain the above
**        copyright notice, this list of conditions and the following
**        disclaimer.
**      * Redistributions in binary form must reproduce the above
**        copyright notice, this list of conditions and the following
**        disclaimer in the documentation and/or other materials provided
**        with the distribution.
**      * Neither the name of the copyright owner nor the names of any
**        other contributors may be used to endorse or promote products
**        derived from this software without specific prior written
**        permission.
**  
**  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
**  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
**  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
**  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
**  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
**  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
**  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
**  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
**  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
**  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
**  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
**
**  CREATION DATE:  21-AUG-1998
**
**  MODIFICATION HISTORY:
**
**  	21-AUG-1992 V1.0    Madison 	Initial coding.
**  	01-SEP-1992 V1.1    Madison 	Comments.
**  	29-SEP-1992 V1.2    Madison 	Separate out command-line symbol defs.
**  	14-OCT-1992 V1.2-1  Madison 	Support use of DCL symbols.
**  	02-JUN-1993 V1.3    Madison 	Support use of $*, $<, etc.
**  	07-JUN-1993 V1.4    Madison 	Add MMS$CHANGED_LIST.
**  	16-SEP-1993 V1.4-1  Madison 	Fix make symbol substitution list.
**  	17-OCT-1993 V1.5    Madison 	Allow $(var:subst-rule) notation.
**  	17-OCT-1993 V1.6    Madison 	Symbol setup for libfile deletion.
**  	28-OCT-1993 V1.6-1  Madison 	Fix redefinition of cmdline symbols.
**  	02-DEC-1993 V1.6-2  Madison 	Allow symbol refs inside symbol refs.
**  	12-DEC-1993 V1.7    Madison 	Support for $(MMS) macro.
**  	02-MAR-1994 V1.7-1  Madison 	Fix non-resolvable specials.
**  	14-APR-1994 V1.7-2  Madison 	Fix MMS$CHANGED_LIST creation.
**  	01-JUL-1994 V1.8    Madison 	CMS support.
**  	22-JUL-1994 V1.9    Madison 	Add MMS$TARGET_FNAME.
**  	17-AUG-1994 V1.9-1  Madison 	Fix Define_Symbol for "MMS" override.
**  	22-AUG-1994 V1.9-2  Madison 	Remove KILL_INTERMEDIATES stuff.
**  	02-DEC-1994 V2.0    Madison 	Add default CMS generation stuff.
**  	21-JUN-1995 V2.1    Madison 	Revamp for MMS parity.
**  	03-OCT-1995 V2.1-1  Madison 	Fix handling of nested symbol refs.
**  	29-MAY-1996 V2.1-2  Madison 	Add MMS$SOURCE_FNAME special symbol.
**  	23-MAR-1997 V2.1-3  Madison 	Fix $(sym:sfx1=sfx) resolution.
**  	27-DEC-1998 V2.2    Madison 	Support version macros; general cleanup.
**      29-JAN-2004 V2.2-1  Madison     Don't define MMS$CMS_LIBRARY as "CMS$LIB".
**	07-APR-2010 V2.3    Sneddon	Added append flag to Define_Symbol plus
**					temporary symbol support in preparation
**					for builtin function support.
**	02-JUL-2012 V2.4    Sneddon	Change to find_char arguments.
**	25-JUL-2012 V3.0    Sneddon	Add builtin function support.
**	28-AUG-2012 V3.0-1  Sneddon	Add ERROR, INFO, WARN, WORDLIST.
**	29-AUG-2012 V3.0-2  Sneddon	Improve WORD and WORDS range checking.
**					 Add FIRSTWORD and LASTWORD. WARNING
**					 alias for WARN.  All builtin file
**					 operations.
**	31-AUG-2012 V3.1    Sneddon	Add support for builtins that handle
**					 their own argument resolution.
**	04-SEP-2012 V3.2    Sneddon	Add OR, AND and IF.
**	07-SEP-2012 V3.3    Sneddon	Add CALL, reorganise temporary symbols.
**	27-SEP-2012 V3.3-2  Sneddon	Add FOREACH.
**	08-OCT-2012 V3.3-3  Sneddon	Add FINDSTRING, FILTER, FILTER-OUT,
**					 STRIP, COLLAPSE.
**--
*/
#pragma module SYMBOLS "V3.3-3"
#include "mmk.h"
#include "globals.h"
#include <stdarg.h>
#include <string.h>
#include <strdef.h>
#include <ots$routines.h>
#include <str$routines.h>
#include <ctype.h>
#include <rms.h>
#define ARGMAX (sizeof(int) * 8)
/*
** Builtin function descriptor
*/
    struct FUNCTION {
	char *name;
	int vararg, maxarg, mask;
	int (*handler)(int, char **, int *);
    };

/*<commit_msg>Forgot to update the ident string.<commit_after>/*
**++
**  FACILITY:	MMK
**
**  ABSTRACT:	Symbol substitution routines
**
**  MODULE DESCRIPTION:
**
**  	This module contains routines that deal with the
**  MMK global and local symbol lists.
**
**  AUTHOR: 	    <NAME>
**
**  Copyright (c) 2008, <NAME>.
**  Copyright (c) 2012, Endless Software Solutions.
**  
**  All rights reserved.
**  
**  Redistribution and use in source and binary forms, with or without
**  modification, are permitted provided that the following conditions
**  are met:
**  
**      * Redistributions of source code must retain the above
**        copyright notice, this list of conditions and the following
**        disclaimer.
**      * Redistributions in binary form must reproduce the above
**        copyright notice, this list of conditions and the following
**        disclaimer in the documentation and/or other materials provided
**        with the distribution.
**      * Neither the name of the copyright owner nor the names of any
**        other contributors may be used to endorse or promote products
**        derived from this software without specific prior written
**        permission.
**  
**  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
**  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
**  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
**  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
**  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
**  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
**  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
**  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
**  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
**  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
**  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
**
**  CREATION DATE:  21-AUG-1998
**
**  MODIFICATION HISTORY:
**
**  	21-AUG-1992 V1.0    Madison 	Initial coding.
**  	01-SEP-1992 V1.1    Madison 	Comments.
**  	29-SEP-1992 V1.2    Madison 	Separate out command-line symbol defs.
**  	14-OCT-1992 V1.2-1  Madison 	Support use of DCL symbols.
**  	02-JUN-1993 V1.3    Madison 	Support use of $*, $<, etc.
**  	07-JUN-1993 V1.4    Madison 	Add MMS$CHANGED_LIST.
**  	16-SEP-1993 V1.4-1  Madison 	Fix make symbol substitution list.
**  	17-OCT-1993 V1.5    Madison 	Allow $(var:subst-rule) notation.
**  	17-OCT-1993 V1.6    Madison 	Symbol setup for libfile deletion.
**  	28-OCT-1993 V1.6-1  Madison 	Fix redefinition of cmdline symbols.
**  	02-DEC-1993 V1.6-2  Madison 	Allow symbol refs inside symbol refs.
**  	12-DEC-1993 V1.7    Madison 	Support for $(MMS) macro.
**  	02-MAR-1994 V1.7-1  Madison 	Fix non-resolvable specials.
**  	14-APR-1994 V1.7-2  Madison 	Fix MMS$CHANGED_LIST creation.
**  	01-JUL-1994 V1.8    Madison 	CMS support.
**  	22-JUL-1994 V1.9    Madison 	Add MMS$TARGET_FNAME.
**  	17-AUG-1994 V1.9-1  Madison 	Fix Define_Symbol for "MMS" override.
**  	22-AUG-1994 V1.9-2  Madison 	Remove KILL_INTERMEDIATES stuff.
**  	02-DEC-1994 V2.0    Madison 	Add default CMS generation stuff.
**  	21-JUN-1995 V2.1    Madison 	Revamp for MMS parity.
**  	03-OCT-1995 V2.1-1  Madison 	Fix handling of nested symbol refs.
**  	29-MAY-1996 V2.1-2  Madison 	Add MMS$SOURCE_FNAME special symbol.
**  	23-MAR-1997 V2.1-3  Madison 	Fix $(sym:sfx1=sfx) resolution.
**  	27-DEC-1998 V2.2    Madison 	Support version macros; general cleanup.
**      29-JAN-2004 V2.2-1  Madison     Don't define MMS$CMS_LIBRARY as "CMS$LIB".
**	07-APR-2010 V2.3    Sneddon	Added append flag to Define_Symbol plus
**					temporary symbol support in preparation
**					for builtin function support.
**	02-JUL-2012 V2.4    Sneddon	Change to find_char arguments.
**	25-JUL-2012 V3.0    Sneddon	Add builtin function support.
**	28-AUG-2012 V3.0-1  Sneddon	Add ERROR, INFO, WARN, WORDLIST.
**	29-AUG-2012 V3.0-2  Sneddon	Improve WORD and WORDS range checking.
**					 Add FIRSTWORD and LASTWORD. WARNING
**					 alias for WARN.  All builtin file
**					 operations.
**	31-AUG-2012 V3.1    Sneddon	Add support for builtins that handle
**					 their own argument resolution.
**	04-SEP-2012 V3.2    Sneddon	Add OR, AND and IF.
**	07-SEP-2012 V3.3    Sneddon	Add CALL, reorganise temporary symbols.
**	27-SEP-2012 V3.3-2  Sneddon	Add FOREACH.
**	08-OCT-2012 V3.3-3  Sneddon	Add FINDSTRING, FILTER, FILTER-OUT,
**					 STRIP, COLLAPSE.
**	24-OCT-2012 V3.3-4  Sneddon	Add JOIN.
**--
*/
#pragma module SYMBOLS "V3.3-4"
#include "mmk.h"
#include "globals.h"
#include <stdarg.h>
#include <string.h>
#include <strdef.h>
#include <ots$routines.h>
#include <str$routines.h>
#include <ctype.h>
#include <rms.h>
#define ARGMAX (sizeof(int) * 8)
/*
** Builtin function descriptor
*/
    struct FUNCTION {
	char *name;
	int vararg, maxarg, mask;
	int (*handler)(int, char **, int *);
    };

/*
{'fraction_non_alphanumeric': 0.10807003421211511, 'fraction_numerical': 0.06429865164016905, 'mean_word_length': 3.1638039379974865, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '587903', 'n_tokens_mistral': 3940, 'n_tokens_neox': 3456, 'n_words': 1158}