<commit_before>#pragma once
/*
* Covariant Script Optimizer
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program.  If not, see <http://www.gnu.org/licenses/>.
*
* Copyright (C) 2017 <NAME>(李登淳)
* Email: <EMAIL>
* Github: https://github.com/mikecovlee
*/
#include "./runtime.hpp"

namespace cs {
	void opt_expr(cov::tree<token_base*>&,cov::tree<token_base*>::iterator);
	bool optimizable(const cov::tree<token_base*>::iterator& it)
	{
		if(!it.usable())
			return false;
		token_base* token=it.data();
		if(token==nullptr)
			return true;
		switch(token->get_type()) {
		default:
			break;
		case token_types::value:
			return true;
			break;
		}
		return false;
	}
	bool constant_var=false;
	bool inside_lambda=false;
	void opt_expr(cov::tree<token_base*>& tree,cov::tree<token_base*>::iterator it)
	{
		if(!it.usable())
			return;
		token_base* token=it.data();
		if(token==nullptr)
			return;
		switch(token->get_type()) {
		default:
			break;
		case token_types::id: {
			const std::string& id=static_cast<token_id*>(token)->get_id();
			if(runtime->constant_storage.var_exsist(id))
				it.data()=new token_value(runtime->constant_storage.get_var(id));
			else if(runtime->storage.var_exsist(id)&&runtime->storage.get_var(id).is_protect())
				it.data()=new token_value(runtime->storage.get_var(id));
			return;
			break;
		}
		case token_types::expr: {
			cov::tree<token_base*>& t=static_cast<token_expr*>(it.data())->get_tree();
			optimize_expression(t);
			if(optimizable(t.root())) {
				it.data()=t.root().data();
			}
			return;
			break;
		}
		case token_types::array: {
			bool is_optimizable=true;
			for(auto& tree:static_cast<token_array*>(token)->get_array()) {
				optimize_expression(tree);
				if(is_optimizable&&!optimizable(tree.root()))
					is_optimizable=false;
			}
			if(is_optimizable) {
				array arr;
				bool is_map=true;
				token_value* t=nullptr;
				for(auto& tree:static_cast<token_array*>(token)->get_array()) {
					const var& val=parse_expr(tree.root());
					if(is_map&&val.type()!=typeid(pair))
						is_map=false;
					arr.push_back((new token_value(copy(val)))->get_value());
				}
				if(arr.empty())
					is_map=false;
				if(is_map) {
					hash_map map;
					for(auto& it:arr) {
						pair& p=it.val<pair>(true);
						if(map.count(p.first)==0)
							map.emplace(p.first,p.second);
						else
							map[p.first]=p.second;
					}
					t=new token_value(var::make<hash_map>(std::move(map)));
				}
				else
					t=new token_value(var::make<array>(std::move(arr)));
				it.data()=t;
			}
			return;
			break;
		}
		case token_types::arglist: {
			for(auto& tree:static_cast<token_arglist*>(token)->get_arglist())
				optimize_expression(tree);
			return;
			break;
		}
		case token_types::signal: {
			switch(static_cast<token_signal*>(token)->get_signal()) {
			default:
				break;
			case signal_types::dot_: {
				opt_expr(tree,it.left());
				opt_expr(tree,it.right());
				token_base* lptr=it.left().data();
				token_base* rptr=it.right().data();<commit_msg>修复new的错误<commit_after>#pragma once
/*
* Covariant Script Optimizer
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program.  If not, see <http://www.gnu.org/licenses/>.
*
* Copyright (C) 2017 <NAME>(李登淳)
* Email: <EMAIL>
* Github: https://github.com/mikecovlee
*/
#include "./runtime.hpp"

namespace cs {
	void opt_expr(cov::tree<token_base*>&,cov::tree<token_base*>::iterator);
	bool optimizable(const cov::tree<token_base*>::iterator& it)
	{
		if(!it.usable())
			return false;
		token_base* token=it.data();
		if(token==nullptr)
			return true;
		switch(token->get_type()) {
		default:
			break;
		case token_types::value:
			return true;
			break;
		}
		return false;
	}
	bool constant_var=false;
	bool inside_lambda=false;
	void opt_expr(cov::tree<token_base*>& tree,cov::tree<token_base*>::iterator it)
	{
		if(!it.usable())
			return;
		token_base* token=it.data();
		if(token==nullptr)
			return;
		switch(token->get_type()) {
		default:
			break;
		case token_types::id: {
			const std::string& id=static_cast<token_id*>(token)->get_id();
			if(runtime->constant_storage.var_exsist(id))
				it.data()=new token_value(runtime->constant_storage.get_var(id));
			else if(runtime->storage.var_exsist(id)&&runtime->storage.get_var(id).is_protect())
				it.data()=new token_value(runtime->storage.get_var(id));
			return;
			break;
		}
		case token_types::expr: {
			cov::tree<token_base*>& t=static_cast<token_expr*>(it.data())->get_tree();
			optimize_expression(t);
			if(optimizable(t.root())) {
				it.data()=t.root().data();
			}
			return;
			break;
		}
		case token_types::array: {
			bool is_optimizable=true;
			for(auto& tree:static_cast<token_array*>(token)->get_array()) {
				optimize_expression(tree);
				if(is_optimizable&&!optimizable(tree.root()))
					is_optimizable=false;
			}
			if(is_optimizable) {
				array arr;
				bool is_map=true;
				token_value* t=nullptr;
				for(auto& tree:static_cast<token_array*>(token)->get_array()) {
					const var& val=parse_expr(tree.root());
					if(is_map&&val.type()!=typeid(pair))
						is_map=false;
					arr.push_back((new token_value(copy(val)))->get_value());
				}
				if(arr.empty())
					is_map=false;
				if(is_map) {
					hash_map map;
					for(auto& it:arr) {
						pair& p=it.val<pair>(true);
						if(map.count(p.first)==0)
							map.emplace(p.first,p.second);
						else
							map[p.first]=p.second;
					}
					t=new token_value(var::make<hash_map>(std::move(map)));
				}
				else
					t=new token_value(var::make<array>(std::move(arr)));
				it.data()=t;
			}
			return;
			break;
		}
		case token_types::arglist: {
			for(auto& tree:static_cast<token_arglist*>(token)->get_arglist())
				optimize_expression(tree);
			return;
			break;
		}
		case token_types::signal: {
			switch(static_cast<token_signal*>(token)->get_signal()) {
			default:
				break;
			case signal_types::new_:
				opt_expr(tree,it.left());
				opt_expr(tree,it.right());
				return;
				break;
			case signal_types::dot_: {
				opt_expr(tree,it.left());
				opt_expr(tree,it.right());
				token_base* lptr=it.left().data();
				token_base* rptr=it.right().data();
{'fraction_non_alphanumeric': 0.1575715282623866, 'fraction_numerical': 0.001674808094905792, 'mean_word_length': 4.111269614835948, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 67, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '728266', 'n_tokens_mistral': 2988, 'n_tokens_neox': 2520, 'n_words': 491}