<commit_before>#include <bitset>
#include <vector>
#include <Refal2.h>

namespace Refal2 {

void CFunctionCompiler::removeCurrentHole()
{
	if( currentHole == firstHole ) {
		firstHole = currentHole->nextHole;
		delete currentHole;
		currentHole = firstHole;
	} else {
		CHole* i = firstHole; 
		while( i->nextHole != currentHole ) {
			i = i->nextHole;
		}
		
		i->nextHole = currentHole->nextHole;
		delete currentHole;
		currentHole = i;
	}
}

void CFunctionCompiler::compileRule(CFunctionRule* rule)
{
	firstHole = new CHole;
	rule->leftPart.Move( &(firstHole->hole) );
	rule->variables.Move( &variables );
	
	do {
		// HSCH
		currentHole = firstHole;
		
		// HSCH 1
		do {
			CUnitNode* left = currentHole->hole.GetFirst();
			CUnitNode* right = currentHole->hole.GetLast();
			
			if( isMarkedVariable(left) )
			{
				// HSCH 2
				classes.push_back( CHolesTuple(
					markedStackDepth[left->Variable()], currentHole ) );
				markedVariables.reset( left->Variable() );
				break;
			} else {
				// HSCH 3
				if( left != right && isFreeVE(left) && isFreeVE(right) ) {
					currentHole = currentHole->nextHole;
				} else {
					matchElement();
					currentHole = firstHole;
				}
			}
		} while( currentHole != 0 );
		
		// HSCH 4
		if( firstHole == 0 ) {
			if( !classes.empty() ) {
				// TODO: EOE( stackDepth - classes.back().stackDepth );
				firstHole = classes.back().firstHole;
				classes.pop_back();
			}
		} else {
			matchVE();
		}
	} while( firstHole != 0 );
}

TVariablesMask CFunctionCompiler::makeVariablesMask(const CHole& hole) const
{
	TVariablesMask variablesMask;
	for( const CUnitNode* i = hole.hole.GetFirst(); i != 0; i = i->Next() ) {
		if( i->IsVariable() && !variables.IsSet( i->Variable() ) ) {
			variablesMask.set( i->Variable() );
		}
	}
	return variablesMask;
}

struct CHoleInfo {
	CHole* hole;
	TVariablesMask variablesMask;
	std::size_t classid;
	
	CHoleInfo(CHole* _hole = 0):
		hole(_hole), classid(0) {}
};

void CFunctionCompiler::splitIntoClasses(CHole* const holes)
{
	std::vector<CHoleInfo> info;	
	for( CHole* i = holes; i != 0; i = i->nextHole ) {
		info.push_back( CHoleInfo(i) );
		info.back().variablesMask = makeVariablesMask(*i);
	}
	
	std::size_t classid = 0;
	for( std::size_t i = 0; i < info.size() - 1; i++ ) {
		if( info[i].classid == 0 ) {
			info[i].classid = ++classid;
			TVariablesMask variablesMask = info[i].variablesMask;
			
			for( std::size_t j = i + 1; j < info.size(); j++ ) {
				if( info[j].classid == 0 &&
					(info[j].variablesMask & variablesMask).any() )
				{
					info[j].classid = classid;
					variablesMask |= info[j].variablesMask;
					j = i;
				}
			}
		}
	}
	
	if( classid > 1 ) {
		for( std::size_t i = 0; i < classid; i++ ) {
			classes.push_back( CHolesTuple(stackDepth) );
		}
		
		std::size_t offset = classes.size() - classid - 1;
		for( std::size_t i = 0; i < info.size(); i++ ) {
			CHole*& tmp = classes[offset + info[i].classid].firstHole;
			info[i].hole->nextHole = tmp;
			tmp = info[i].hole;
		}
	} else {
		classes.push_back( CHolesTuple(stackDepth, holes) );
	}
}

void CFunctionCompiler::matchVE()
{
	// TODO: action
	printf("CFunctionCompiler::matchVE()\n");
	currentHole = firstHole;
	delete detachLeftUnitInCurrentHole();
}

void CFunctionCompiler::matchElement()
{
	//if( currentLeft != currentHole->left || currentRight != currentHole->Right ) {
		/* TODO: SB( currentHole->left, currentHole->Right ) */
	//}
	
	if( currentHole->hole.IsEmpty() ) {
		matchEmptyExpression();
	} else {
		CUnitNode* left = currentHole->hole.GetFirst();
		CUnitNode* right = currentHole->hole.GetLast();
	
		if( left == left && isFreeVE(left) ) {
			matchClosedE();
		} else if( left->IsParen() ) {
			matchLeftParens();
		} else if( left->IsSymbol() ) {
			matchLeftSymbol();
		} else if( left->IsVariable() && tryMatchLeftVariable( left ) ) {
		} else if( right->IsParen() ) {
			matchRightParens();
		} else if( right->IsSymbol() ) {
			matchRightSymbol();<commit_msg>small error fixes<commit_after>#include <bitset>
#include <vector>
#include <Refal2.h>

namespace Refal2 {

void CFunctionCompiler::removeCurrentHole()
{
	if( currentHole == firstHole ) {
		firstHole = currentHole->nextHole;
		delete currentHole;
		currentHole = firstHole;
	} else {
		CHole* i = firstHole; 
		while( i->nextHole != currentHole ) {
			i = i->nextHole;
		}
		
		i->nextHole = currentHole->nextHole;
		delete currentHole;
		currentHole = i;
	}
}

void CFunctionCompiler::compileRule(CFunctionRule* rule)
{
	firstHole = new CHole;
	rule->leftPart.Move( &(firstHole->hole) );
	rule->variables.Move( &variables );
	
	do {
		// HSCH
		currentHole = firstHole;
		
		// HSCH 1
		do {
			CUnitNode* left = currentHole->hole.GetFirst();
			CUnitNode* right = currentHole->hole.GetLast();
			
			if( isMarkedVariable(left) )
			{
				// HSCH 2
				classes.push_back( CHolesTuple(
					markedStackDepth[left->Variable()], currentHole ) );
				markedVariables.reset( left->Variable() );
				break;
			} else {
				// HSCH 3
				if( left != right && isFreeVE(left) && isFreeVE(right) ) {
					currentHole = currentHole->nextHole;
				} else {
					matchElement();
					currentHole = firstHole;
				}
			}
		} while( currentHole != 0 );
		
		// HSCH 4
		if( firstHole == 0 ) {
			if( !classes.empty() ) {
				// TODO: EOE( stackDepth - classes.back().stackDepth );
				firstHole = classes.back().firstHole;
				classes.pop_back();
			}
		} else {
			matchVE();
		}
	} while( firstHole != 0 );
}

TVariablesMask CFunctionCompiler::makeVariablesMask(const CHole& hole) const
{
	TVariablesMask variablesMask;
	for( const CUnitNode* i = hole.hole.GetFirst(); i != 0; i = i->Next() ) {
		if( i->IsVariable() && !variables.IsSet( i->Variable() ) ) {
			variablesMask.set( i->Variable() );
		}
	}
	return variablesMask;
}

struct CHoleInfo {
	CHole* hole;
	TVariablesMask variablesMask;
	std::size_t classid;
	
	CHoleInfo(CHole* _hole = 0):
		hole(_hole), classid(0) {}
};

void CFunctionCompiler::splitIntoClasses(CHole* const holes)
{
	std::vector<CHoleInfo> info;	
	for( CHole* i = holes; i != 0; i = i->nextHole ) {
		info.push_back( CHoleInfo(i) );
		info.back().variablesMask = makeVariablesMask(*i);
	}
	
	std::size_t classid = 0;
	for( std::size_t i = 0; i < info.size() - 1; i++ ) {
		if( info[i].classid == 0 ) {
			info[i].classid = ++classid;
			TVariablesMask variablesMask = info[i].variablesMask;
			
			for( std::size_t j = i + 1; j < info.size(); j++ ) {
				if( info[j].classid == 0 &&
					(info[j].variablesMask & variablesMask).any() )
				{
					info[j].classid = classid;
					variablesMask |= info[j].variablesMask;
					j = i;
				}
			}
		}
	}
	
	if( classid > 1 ) {
		for( std::size_t i = 0; i < classid; i++ ) {
			classes.push_back( CHolesTuple(stackDepth) );
		}
		
		std::size_t offset = classes.size() - classid - 1;
		for( std::size_t i = 0; i < info.size(); i++ ) {
			CHole*& tmp = classes[offset + info[i].classid].firstHole;
			info[i].hole->nextHole = tmp;
			tmp = info[i].hole;
		}
	} else {
		classes.push_back( CHolesTuple(stackDepth, holes) );
	}
}

void CFunctionCompiler::matchVE()
{
	// TODO: action
	printf("CFunctionCompiler::matchVE()\n");
	currentHole = firstHole;
	delete detachLeftUnitInCurrentHole();
}

void CFunctionCompiler::matchElement()
{
	//if( currentLeft != currentHole->left || currentRight != currentHole->Right ) {
		/* TODO: SB( currentHole->left, currentHole->Right ) */
	//}
	
	if( currentHole->hole.IsEmpty() ) {
		matchEmptyExpression();
	} else {
		CUnitNode* left = currentHole->hole.GetFirst();
		CUnitNode* right = currentHole->hole.GetLast();
	
		if( left == right && isFreeVE(left) ) {
			matchClosedE();
		} else if( left->IsParen() ) {
			matchLeftParens();
		} else if( left->IsSymbol() ) {
			matchLeftSymbol();
		} else if( left->IsVariable() && tryMatchLeftVariable( left ) ) {
		} else if( right->IsParen() ) {
			matchRightParens();
		} else if( right->IsSymbol() ) {
			matchRightSymbol();
{'fraction_non_alphanumeric': 0.1671498802470692, 'fraction_numerical': 0.005798562964830455, 'mean_word_length': 3.3858485351022662, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 75, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7427415', 'n_tokens_mistral': 3354, 'n_tokens_neox': 2916, 'n_words': 579}