<commit_before>/*
 *  Copyright 2007 <NAME> <<EMAIL>>
 */

/*  This file is part of Colm.
 *
 *  Colm is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 * 
 *  Colm is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 * 
 *  You should have received a copy of the GNU General Public License
 *  along with Colm; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA 
 */

#include "bytecode.h"
#include "astring.h"
#include "pdarun.h"
#include "dlistval.h"
#include "fsmrun.h"
#include "pdarun.h"
#include <iostream>
#include <sstream>
#include <alloca.h>
#include <sys/mman.h>

using std::cout;
using std::cerr;
using std::endl;

#define push(i) (*(--sp) = (i))
#define pop() (*sp++)
#define top() (*sp)
#define ptop() (sp)
#define popn(n) (sp += (n))
#define pushn(n) (sp -= (n))
#define local(o) (frame[o])
#define plocal(o) (&frame[o])
#define local_iframe(o) (iframe[o])
#define plocal_iframe(o) (&iframe[o])

#define read_byte( i ) do { \
	i = ((uchar) *instr++); \
} while(0)

#define read_word_p( i, p ) do { \
	i = ((Word)  p[0]); \
	i |= ((Word) p[1]) << 8; \
	i |= ((Word) p[2]) << 16; \
	i |= ((Word) p[3]) << 24; \
} while(0)

#define read_word( i ) do { \
	i = ((Word) *instr++); \
	i |= ((Word) *instr++) << 8; \
	i |= ((Word) *instr++) << 16; \
	i |= ((Word) *instr++) << 24; \
} while(0)

#define read_tree( i ) do { \
	Word w; \
	w = ((Word) *instr++); \
	w |= ((Word) *instr++) << 8; \
	w |= ((Word) *instr++) << 16; \
	w |= ((Word) *instr++) << 24; \
	i = (Tree*) w; \
} while(0)

#define read_half( i ) do { \
	i = ((Word) *instr++); \
	i |= ((Word) *instr++) << 8; \
} while(0)

int colm_log_bytecode = 0;
int colm_log_parse = 0;
int colm_log_match = 0;
int colm_log_compile = 0;
int colm_log_conds = 0;

Tree *prep_parse_tree( Program *prg, Tree **sp, Tree *tree )
{
	/* If the tree was produced by a parsing function then we need to send a
	 * copy of it because the parsing that we are about to do requires fresh
	 * parsing algorithm data. */
	if ( !(tree->flags & AF_PARSE_TREE) || tree->flags & AF_PARSED ) {
		#ifdef COLM_LOG_BYTECODE
		if ( colm_log_bytecode ) {
			cerr << "copying tree in send function" << endl;
		}
		#endif
		Kid *unused = 0;
		Tree *newTree = copy_real_tree( prg, tree, 0, unused, true );
		tree_upref( newTree );

		tree_downref( prg, sp, tree );

		tree = newTree;
	}
	return  tree;
}

void send( Tree **root, Program *prg, PdaRun *parser, Tree *tree, bool ignore )
{
	tree = prep_parse_tree( prg, root, tree );

	if ( tree->id >= prg->rtd->firstNonTermId )
		tree->id = prg->rtd->lelInfo[tree->id].termDupId;

	tree->flags |= AF_ARTIFICIAL;

	/* FIXME: Do we need to remove the ignore tokens 
	 * at this point? Will it cause a leak? */

	Kid *kid = prg->kidPool.allocate();
	kid->tree = tree;

	if ( parser->queue == 0 )
		parser->queue = parser->queueLast = kid;
	else {
		parser->queueLast->next = kid;
		parser->queueLast = kid;
	}
}

Tree *call_parser( Tree **&sp, Program *prg, Stream *stream, 
		long parserId, long stopId, CodeVect *&cv, bool revertOn )
{
	PdaTables *tables = prg->rtd->pdaTables;
	PdaRun parser( sp, prg, tables, parserId, stream->fsmRun, stopId, revertOn );
	stream->scanner->run( &parser );
	commit_full( &parser, 0 );
	Tree *tree = parser.getParsedRoot( stopId > 0 );
	tree_upref( tree );
	parser.clean();

	/* Maybe return the reverse code. */
	if ( revertOn )
		cv = parser.allReverseCode;
	else {
		delete parser.allReverseCode;
		cv = 0;
	}

	/* Indicate that this tree came out of a parser. */
	tree->flags |= AF_PARSED;

	return tree;
}

void undo_parse( Tree **&sp, Program *prg, Stream *stream, 
		long parserId, Tree *tree, CodeVect *rev )
{
	PdaTables *tables = prg->rtd->pdaTables;
	PdaRun parser( sp, prg, tables, parserId, stream->fsmRun, 0, false );
	parser.undoParse( tree, rev );
}

Tree *stream_pull( Program *prg, Stream *stream, Tree *length )
{
	long len = ((Int*)length)->value;<commit_msg>Fixed broken build from last commit.<commit_after>/*
 *  Copyright 2007 <NAME> <<EMAIL>>
 */

/*  This file is part of Colm.
 *
 *  Colm is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 * 
 *  Colm is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 * 
 *  You should have received a copy of the GNU General Public License
 *  along with Colm; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA 
 */

#include "bytecode.h"
#include "astring.h"
#include "pdarun.h"
#include "dlistval.h"
#include "fsmrun.h"
#include "pdarun.h"
#include <iostream>
#include <sstream>
#include <alloca.h>
#include <sys/mman.h>

using std::cout;
using std::cerr;
using std::endl;

#define push(i) (*(--sp) = (i))
#define pop() (*sp++)
#define top() (*sp)
#define ptop() (sp)
#define popn(n) (sp += (n))
#define pushn(n) (sp -= (n))
#define local(o) (frame[o])
#define plocal(o) (&frame[o])
#define local_iframe(o) (iframe[o])
#define plocal_iframe(o) (&iframe[o])

#define read_byte( i ) do { \
	i = ((uchar) *instr++); \
} while(0)

#define read_word_p( i, p ) do { \
	i = ((Word)  p[0]); \
	i |= ((Word) p[1]) << 8; \
	i |= ((Word) p[2]) << 16; \
	i |= ((Word) p[3]) << 24; \
} while(0)

#define read_word( i ) do { \
	i = ((Word) *instr++); \
	i |= ((Word) *instr++) << 8; \
	i |= ((Word) *instr++) << 16; \
	i |= ((Word) *instr++) << 24; \
} while(0)

#define read_tree( i ) do { \
	Word w; \
	w = ((Word) *instr++); \
	w |= ((Word) *instr++) << 8; \
	w |= ((Word) *instr++) << 16; \
	w |= ((Word) *instr++) << 24; \
	i = (Tree*) w; \
} while(0)

#define read_half( i ) do { \
	i = ((Word) *instr++); \
	i |= ((Word) *instr++) << 8; \
} while(0)

int colm_log_bytecode = 0;
int colm_log_parse = 0;
int colm_log_match = 0;
int colm_log_compile = 0;
int colm_log_conds = 0;

Tree *prep_parse_tree( Program *prg, Tree **sp, Tree *tree )
{
	/* If the tree was produced by a parsing function then we need to send a
	 * copy of it because the parsing that we are about to do requires fresh
	 * parsing algorithm data. */
	if ( !(tree->flags & AF_PARSE_TREE) || tree->flags & AF_PARSED ) {
		#ifdef COLM_LOG_BYTECODE
		if ( colm_log_bytecode ) {
			cerr << "copying tree in send function" << endl;
		}
		#endif
		Kid *unused = 0;
		Tree *newTree = copy_real_tree( prg, tree, 0, unused, true );
		tree_upref( newTree );

		tree_downref( prg, sp, tree );

		tree = newTree;
	}
	return  tree;
}

void send( Tree **root, Program *prg, PdaRun *parser, Tree *tree, bool ignore )
{
	tree = prep_parse_tree( prg, root, tree );

	if ( tree->id >= prg->rtd->firstNonTermId )
		tree->id = prg->rtd->lelInfo[tree->id].termDupId;

	tree->flags |= AF_ARTIFICIAL;

	/* FIXME: Do we need to remove the ignore tokens 
	 * at this point? Will it cause a leak? */

	Kid *kid = prg->kidPool.allocate();
	kid->tree = tree;

	if ( parser->queue == 0 )
		parser->queue = parser->queueLast = kid;
	else {
		parser->queueLast->next = kid;
		parser->queueLast = kid;
	}
}

Tree *call_parser( Tree **&sp, Program *prg, Stream *stream, 
		long parserId, long stopId, CodeVect *&cv, bool revertOn )
{
	PdaTables *tables = prg->rtd->pdaTables;
	PdaRun parser( sp, prg, tables, parserId, stream->fsmRun, stopId, revertOn );
	stream->fsmRun->run( &parser );
	commit_full( &parser, 0 );
	Tree *tree = parser.getParsedRoot( stopId > 0 );
	tree_upref( tree );
	parser.clean();

	/* Maybe return the reverse code. */
	if ( revertOn )
		cv = parser.allReverseCode;
	else {
		delete parser.allReverseCode;
		cv = 0;
	}

	/* Indicate that this tree came out of a parser. */
	tree->flags |= AF_PARSED;

	return tree;
}

void undo_parse( Tree **&sp, Program *prg, Stream *stream, 
		long parserId, Tree *tree, CodeVect *rev )
{
	PdaTables *tables = prg->rtd->pdaTables;
	PdaRun parser( sp, prg, tables, parserId, stream->fsmRun, 0, false );
	parser.undoParse( tree, rev );
}

Tree *stream_pull( Program *prg, Stream *stream, Tree *length )
{
	long len = ((Int*)length)->value;
{'fraction_non_alphanumeric': 0.1671280276816609, 'fraction_numerical': 0.012918108419838524, 'mean_word_length': 3.5805599577390383, 'pattern_counts': {'":': 0, '<': 65, '<?xml version=': 0, '>': 77, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2309467', 'n_tokens_mistral': 3433, 'n_tokens_neox': 3247, 'n_words': 1052}