<commit_before>
import builtins

from cpp.cpprenderer import EeyCppRenderer
from environment import EeyEnvironment
from values import *

def parse_tree_string_to_values( string ):
	return eval( string )

def non_empty_line( ln ):
	return ( ln.strip() != "" )

def parse_tree_to_cpp( parse_tree_in_fl, cpp_out_fl ):
	env = EeyEnvironment( EeyCppRenderer() )
	builtins.add_builtins( self )

	values = ( parse_tree_string_to_values( ln ) for ln in
		filter( non_empty_line, parse_tree_in_fl ) )

	cpp_out_fl.write( env.render_exe( values ) )

<commit_msg>Handle comments in parse tree.<commit_after>from itertools import imap

import builtins

from cpp.cpprenderer import EeyCppRenderer
from environment import EeyEnvironment
from functionvalues import *
from languagevalues import *
from values import *

def parse_tree_string_to_values( string ):
	return eval( string )

def remove_comments( ln ):
	i = ln.find( "#" )
	if i != -1:
		return ln[:i]
	else:
		return ln

def non_empty_line( ln ):
	return ( ln.strip() != "" )

def parse_tree_to_cpp( parse_tree_in_fl, cpp_out_fl ):
	env = EeyEnvironment( EeyCppRenderer() )
	builtins.add_builtins( env )

	values = ( parse_tree_string_to_values( ln ) for ln in
		filter( non_empty_line, imap( remove_comments, parse_tree_in_fl ) ) )

	cpp_out_fl.write( env.render_exe( values ) )


{'fraction_non_alphanumeric': 0.09049773755656108, 'fraction_numerical': 0.0007541478129713424, 'mean_word_length': 5.059360730593608, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7538562', 'n_tokens_mistral': 483, 'n_tokens_neox': 466, 'n_words': 120}