<commit_before>/* -MODULE----------------------------------------------------------------------
Phorward Foundation Toolkit
Copyright (C) 2006-2018 by Phorward Software Technologies, <NAME>
http://www.phorward-software.com ++ contact<at>phorward<dash>software<dot>com
All rights reserved. See LICENSE for more information.

File:	pbnf.c
Usage:	Parsing full parser definitions from Phorward BNF (PBNF).
----------------------------------------------------------------------------- */

#include "phorward.h"

#define NAMELEN			72
#define DERIVCHAR		'\''

static pboolean traverse_production( ppgram* gram, ppsym* lhs, ppast* node );

/* Derive name from basename */
static char* derive_name( ppgram* gram, char* base )
{
	int             i;
	static char		deriv   [ ( NAMELEN * 2 ) + 1 ];

	sprintf( deriv, "%s%c", base, DERIVCHAR );

	for( i = 0; strlen( deriv ) < ( NAMELEN * 2 ); i++ )
	{
		if( !pp_sym_get_by_name( gram, deriv ) )
			return deriv;

		sprintf( deriv + strlen( deriv ), "%c", DERIVCHAR );
	}

	return (char*)NULL;
}

#define NODE_IS( n, s ) 	( !strcmp( (n)->emit, s ) )

static ppsym* traverse_terminal( ppgram* gram, ppast* node )
{
	ppsym*			sym;
	pregex_ptn*		ptn			= (pregex_ptn*)NULL;
	char			name		[ NAMELEN * 2 + 1 ];
	char			ch;

	sprintf( name, "%.*s", (int)node->len, node->start );

	if( !( sym = pp_sym_get_by_name( gram, name ) ) )
	{
		if( NODE_IS( node, "CCL" ) )
			ptn = pregex_ptn_create( name, PREGEX_COMP_NOERRORS );
		else if( !NODE_IS( node, "Terminal" ) )
		{
			ch = name[ pstrlen( name ) - 1 ];
			name[ pstrlen( name ) - 1 ] = '\0';

			if( NODE_IS( node, "Token") || NODE_IS( node, "String" ) )
				ptn = pregex_ptn_create( name + 1, PREGEX_COMP_STATIC );
			else if( NODE_IS( node, "Regex" ) )
				ptn = pregex_ptn_create( name + 1, PREGEX_COMP_NOERRORS );
			else
				MISSINGCASE;

			name[ pstrlen( name ) ] = ch;
		}

		sym = pp_sym_create( gram, name,
				PPFLAG_FREENAME | PPFLAG_DEFINED );
		sym->ptn = ptn;

		if( NODE_IS( node, "Token") )
		{
			sym->emit = pstrndup( sym->name + 1,
							strlen( sym->name ) - 2 );
			sym->flags |= PPFLAG_FREEEMIT;
		}
	}

	return sym;
}

static ppsym* traverse_symbol( ppgram* gram, ppsym* lhs, ppast* node )
{
	ppsym*			sym;
	ppast*			child;
	char			name		[ NAMELEN * 2 + 1 ];

	/* fprintf( stderr, "sym >%s<\n", node->emit ); */

	if( NODE_IS( node, "inline") )
	{
		sym = pp_sym_create( gram, derive_name( gram, lhs->name ),
									PPFLAG_FREENAME | PPFLAG_DEFINED
										| PPFLAG_GENERATED );

		for( child = node->child; child; child = child->next )
			if( !traverse_production( gram, sym, child->child ) )
				return (ppsym*)NULL;
	}
	else
	{
		if( NODE_IS( node, "Nonterminal") )
		{
			sprintf( name, "%.*s", (int)node->len, node->start );

			if( !( sym = pp_sym_get_by_name( gram, name ) ) )
				sym = pp_sym_create( gram, name, PPFLAG_FREENAME );
		}
		else
			sym = traverse_terminal( gram, node );
	}

	if( sym )
		sym->flags |= PPFLAG_CALLED;

	return sym;
}


static pboolean traverse_production( ppgram* gram, ppsym* lhs, ppast* node )
{
	ppsym*			sym;
	ppsym*			csym;
	ppprod*			prod;
	ppprod*			popt;
	int				i;

	prod = pp_prod_create( gram, lhs, (ppsym*)NULL );

	for( ; node; node = node->next )
	{
		/* fprintf( stderr, "prod >%s<\n", node->emit ); */

		if( NODE_IS( node, "symbol" ) )
		{
			if( !( sym = traverse_symbol( gram, lhs, node->child ) ) )
				return FALSE;

			pp_prod_append( prod, sym );
		}
		else if( NODE_IS( node, "emits" ) )
		{
			if( NODE_IS( node->child, "Code" ) )
				prod->emit = pstrndup( node->child->start + 2,
										node->child->len - 4 );
			else
				prod->emit = pstrndup( node->child->start, node->child->len );

			prod->flags |= PPFLAG_FREEEMIT;
		}
		else
		{
			sym = traverse_symbol( gram, lhs, node->child );

			if( NODE_IS( node, "pos" ) )
				sym = pp_sym_mod_positive( sym );

			else if( NODE_IS( node, "opt" ) )
				sym = pp_sym_mod_optional( sym );

			else if( NODE_IS( node, "kle" ) )
				sym = pp_sym_mod_kleene( sym );

			pp_prod_append( prod, sym );
		}
	}

	/*
		Optimize productions with one generated symbol
		that was introduced via 'inline'.
	*/
	if( ( sym = pp_prod_getfromrhs( prod, 0 ) )
			&& !pp_prod_getfromrhs( prod, 1 ) )
	{
		if( !PPSYM_IS_TERMINAL( sym )
				&& sym->flags & PPFLAG_GENERATED
					&& sym->emit && !pp_sym_getprod( sym, 1 ) )
		{
			if( sym->flags & PPFLAG_FREEEMIT )
			{
				prod->emit = pstrdup( sym->emit );
				prod->flags |= PPFLAG_FREEEMIT;
			}
			else
				prod->emit = sym->emit;

			pp_prod_remove( prod, sym );

			popt = pp_sym_getprod( sym, 0 );

			for( i = 0; ( csym = pp_prod_getfromrhs( popt, i ) ); i++ )
				pp_prod_append( prod, csym );

			pp_sym_drop( sym );
		}
	}

	return TRUE;
}

static pboolean ast_to_gram( ppgram* gram, ppast* ast )
{
	ppsym*			sym;
	ppsym*			nonterm		= (ppsym*)NULL;
	ppast* 			node;
	ppast*			child;
	char			name		[ NAMELEN * 2 + 1 ];
	char			def			[ NAMELEN * 2 + 1 ];
	char*			emit;
	pregex_ptn*		ptn;
	pboolean		flag_ignore;
	ppassoc			assoc;
	unsigned int	prec		= 0;

	/* pp_ast_dump_short( stderr, ast ); */

	for( node = ast; node; node = node->next )
	{
		emit = (char*)NULL;
		flag_ignore = FALSE;

		/* fprintf( stderr, "gram >%s<\n", node->emit ); */

		if( NODE_IS( node, "nontermdef" ) )
		{<commit_msg>Fixing some pbnf issues<commit_after>/* -MODULE----------------------------------------------------------------------
Phorward Foundation Toolkit
Copyright (C) 2006-2018 by Phorward Software Technologies, <NAME>
http://www.phorward-software.com ++ contact<at>phorward<dash>software<dot>com
All rights reserved. See LICENSE for more information.

File:	pbnf.c
Usage:	Parsing full parser definitions from Phorward BNF (PBNF).
----------------------------------------------------------------------------- */

#include "phorward.h"

#define NAMELEN			72
#define DERIVCHAR		'\''

static pboolean traverse_production( ppgram* gram, ppsym* lhs, ppast* node );

/* Derive name from basename */
static char* derive_name( ppgram* gram, char* base )
{
	int             i;
	static char		deriv   [ ( NAMELEN * 2 ) + 1 ];

	sprintf( deriv, "%s%c", base, DERIVCHAR );

	for( i = 0; strlen( deriv ) < ( NAMELEN * 2 ); i++ )
	{
		if( !pp_sym_get_by_name( gram, deriv ) )
			return deriv;

		sprintf( deriv + strlen( deriv ), "%c", DERIVCHAR );
	}

	return (char*)NULL;
}

#define NODE_IS( n, s ) 	( !strcmp( (n)->emit, s ) )

static ppsym* traverse_terminal( ppgram* gram, ppast* node )
{
	ppsym*			sym;
	pregex_ptn*		ptn			= (pregex_ptn*)NULL;
	char			name		[ NAMELEN * 2 + 1 ];
	char			ch;

	PROC( "traverse_terminal" );
	VARS( "node->emit", "%s", node->emit );

	sprintf( name, "%.*s", (int)node->len, node->start );
	VARS( "name", "%s", name );

	if( !( sym = pp_sym_get_by_name( gram, name ) ) )
	{
		if( NODE_IS( node, "CCL" ) )
			ptn = pregex_ptn_create( name, PREGEX_COMP_NOERRORS );
		else if( !NODE_IS( node, "Terminal" ) )
		{
			ch = name[ pstrlen( name ) - 1 ];
			name[ pstrlen( name ) - 1 ] = '\0';

			if( NODE_IS( node, "Token") || NODE_IS( node, "String" ) )
				ptn = pregex_ptn_create( name + 1, PREGEX_COMP_STATIC );
			else if( NODE_IS( node, "Regex" ) )
				ptn = pregex_ptn_create( name + 1, PREGEX_COMP_NOERRORS );
			else
				MISSINGCASE;

			name[ pstrlen( name ) ] = ch;
		}

		sym = pp_sym_create( gram, name,
				PPFLAG_FREENAME | PPFLAG_DEFINED );
		sym->ptn = ptn;

		if( NODE_IS( node, "Token") )
		{
			sym->emit = pstrndup( sym->name + 1,
							strlen( sym->name ) - 2 );
			sym->flags |= PPFLAG_FREEEMIT;
		}
	}

	RETURN( sym );
}

static ppsym* traverse_symbol( ppgram* gram, ppsym* lhs, ppast* node )
{
	ppsym*			sym;
	ppast*			child;
	char			name		[ NAMELEN * 2 + 1 ];

	PROC( "traverse_symbol" );
	PARMS( "node->emit", "%s", node->emit );

	if( NODE_IS( node, "inline" ) )
	{
		sym = pp_sym_create( gram, derive_name( gram, lhs->name ),
									PPFLAG_FREENAME | PPFLAG_DEFINED
										| PPFLAG_GENERATED );

		for( child = node->child; child; child = child->next )
			if( !traverse_production( gram, sym, child->child ) )
				RETURN( (ppsym*)NULL );
	}
	else
	{
		if( NODE_IS( node, "Nonterminal" ) )
		{
			sprintf( name, "%.*s", (int)node->len, node->start );

			if( !( sym = pp_sym_get_by_name( gram, name ) ) )
				sym = pp_sym_create( gram, name, PPFLAG_FREENAME );
		}
		else
			sym = traverse_terminal( gram, node );
	}

	if( sym )
		sym->flags |= PPFLAG_CALLED;

	RETURN( sym );
}


static pboolean traverse_production( ppgram* gram, ppsym* lhs, ppast* node )
{
	ppsym*			sym;
	ppsym*			csym;
	ppprod*			prod;
	ppprod*			popt;
	int				i;

	PROC( "traverse_production" );

	prod = pp_prod_create( gram, lhs, (ppsym*)NULL );

	for( ; node; node = node->next )
	{
		VARS( "node->emit", "%s", node->emit );

		if( NODE_IS( node, "symbol" ) )
		{
			if( !( sym = traverse_symbol( gram, lhs, node->child ) ) )
				RETURN( FALSE );

			pp_prod_append( prod, sym );
		}
		else if( NODE_IS( node, "emits" ) )
		{
			if( NODE_IS( node->child, "Code" ) )
				prod->emit = pstrndup( node->child->start + 2,
										node->child->len - 4 );
			else
				prod->emit = pstrndup( node->child->start, node->child->len );

			prod->flags |= PPFLAG_FREEEMIT;
		}
		else
		{
			sym = traverse_symbol( gram, lhs, node->child->child );

			if( NODE_IS( node, "pos" ) )
				sym = pp_sym_mod_positive( sym );

			else if( NODE_IS( node, "opt" ) )
				sym = pp_sym_mod_optional( sym );

			else if( NODE_IS( node, "kle" ) )
				sym = pp_sym_mod_kleene( sym );

			pp_prod_append( prod, sym );
		}
	}

	/*
		Optimize productions with one generated symbol
		that was introduced via 'inline'.
	*/
	if( ( sym = pp_prod_getfromrhs( prod, 0 ) )
			&& !pp_prod_getfromrhs( prod, 1 ) )
	{
		if( !PPSYM_IS_TERMINAL( sym )
				&& sym->flags & PPFLAG_GENERATED
					&& sym->emit && !pp_sym_getprod( sym, 1 ) )
		{
			if( sym->flags & PPFLAG_FREEEMIT )
			{
				prod->emit = pstrdup( sym->emit );
				prod->flags |= PPFLAG_FREEEMIT;
			}
			else
				prod->emit = sym->emit;

			pp_prod_remove( prod, sym );

			popt = pp_sym_getprod( sym, 0 );

			for( i = 0; ( csym = pp_prod_getfromrhs( popt, i ) ); i++ )
				pp_prod_append( prod, csym );

			pp_sym_drop( sym );
		}
	}

	RETURN( TRUE );
}

static pboolean ast_to_gram( ppgram* gram, ppast* ast )
{
	ppsym*			sym;
	ppsym*			nonterm		= (ppsym*)NULL;
	ppast* 			node;
	ppast*			child;
	char			name		[ NAMELEN * 2 + 1 ];
	char			def			[ NAMELEN * 2 + 1 ];
	char*			emit;
	pregex_ptn*		ptn;
	pboolean		flag_ignore;
	ppassoc			assoc;
	unsigned int	prec		= 0;

	/* pp_ast_dump_short( stderr, ast ); */

	for( node = ast; node; node = node->next )
	{
		emit = (char*)NULL;
		flag_ignore = FALSE;

		/* fprintf( stderr, "gram >%s<\n", node->emit ); */

		if( NODE_IS( node, "nontermdef" ) )
		{
{'fraction_non_alphanumeric': 0.182794694563796, 'fraction_numerical': 0.006912011955912572, 'mean_word_length': 2.661764705882353, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 104, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7173555', 'n_tokens_mistral': 4876, 'n_tokens_neox': 4294, 'n_words': 1033}