<commit_before><?php

class FormalTheory_RegularExpression_Token_Union extends FormalTheory_RegularExpression_Token
{
	
	private $_regex_array;
	
	function __construct( array $regex_array )
	{
		foreach( $regex_array as $regex ) {
			if( ! $regex instanceof FormalTheory_RegularExpression_Token_Regex ) {
				throw new RuntimeException( "union can only take regex: ".var_export( $regex, TRUE ) );
			}
		}
		$this->_regex_array = $regex_array;
	}
	
	function __toString()
	{
		
		return implode( "|", array_map( function( $regex ) {
			$regex_tokens = $regex->getTokens();
			if( count( $regex_tokens ) === 1 && $regex_tokens[0] instanceof FormalTheory_RegularExpression_Token_Union ) {
				return substr( $regex, 1, -1 );
			}
			return (string)$regex;
		}, $this->_regex_array ) );
	}
	
	function getRegexArray()
	{<commit_msg>Allow Union to hold non-regex tokens.<commit_after><?php

class FormalTheory_RegularExpression_Token_Union extends FormalTheory_RegularExpression_Token
{
	
	private $_regex_array;
	
	function __construct( array $regex_array )
	{
		foreach( $regex_array as $regex ) {
			if( ! $regex instanceof FormalTheory_RegularExpression_Token ) {
				throw new RuntimeException( "union can only take tokens: ".var_export( $regex, TRUE ) );
			}
		}
		$this->_regex_array = array_values( $regex_array );
	}
	
	function __toString()
	{
		return implode( "|", $this->_regex_array );
	}
	
	function getTokens()
	{
{'fraction_non_alphanumeric': 0.1150070126227209, 'fraction_numerical': 0.002805049088359046, 'mean_word_length': 4.57421875, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '820236', 'n_tokens_mistral': 513, 'n_tokens_neox': 454, 'n_words': 103}