<commit_before><?php

namespace alojaweb\Controller;

use alojaweb\inc\HighCharts;
use alojaweb\inc\Utils;
use alojaweb\inc\DBUtils;
use alojaweb\inc\MLUtils;

class MLParamevalController extends AbstractController
{
	public function mlparamEvaluationAction()
	{
		$db = $this->container->getDBUtils ();
		$rows = $categories = $series = '';
		try {
			$configurations = array ();	// Useless here
			$where_configs = '';
			$concat_config = ""; 		// Useless here
			
			$params = array();
			$param_names = array('benchs','nets','disks','mapss','iosfs','replications','iofilebufs','comps','blk_sizes','id_clusters'); // Order is important
			foreach ($param_names as $p) { $params[$p] = Utils::read_params($p,$where_configs,$configurations,$concat_config); sort($params[$p]); }

			if (count($_GET) <= 1
			|| (count($_GET) == 2 && array_key_exists('parameval',$_GET))
			|| (count($_GET) == 2 && array_key_exists('current_model',$_GET)))
			{
				$params['benchs'] = array('terasort');
				$where_configs = ' AND bench IN ("terasort")';
				if (!isset($_GET['parameval']) || $_GET['parameval'] != 'net') $params['nets'] = array('ETH'); $where_configs .= ' AND net IN ("ETH")';
				if (!isset($_GET['parameval']) || $_GET['parameval'] != 'disk') $params['disks'] = array('HDD','SSD'); $where_configs .= ' AND disk IN ("HDD","SSD")';
				if (!isset($_GET['parameval']) || $_GET['parameval'] != 'iofilebuf') $params['iofilebufs'] = array('32768','65536','131072'); $where_configs .= ' AND iofilebuf IN ("32768","65536","131072")';
				if (!isset($_GET['parameval']) || $_GET['parameval'] != 'iofs') $params['iosfs'] = array('10'); $where_configs .= ' AND iosf IN ("10")';
				if (!isset($_GET['parameval']) || $_GET['parameval'] != 'comp') $params['comps'] = array('0'); $where_configs .= ' AND comp IN ("0")';
				if (!isset($_GET['parameval']) || $_GET['parameval'] != 'replication') $params['replications'] = array('1'); $where_configs .= ' AND replication IN ("1")';
			}

			$money		= Utils::read_params ( 'money', $where_configs, $configurations, $concat_config );
			$paramEval	= (isset($_GET['parameval']) && $_GET['parameval'] != '') ? $_GET['parameval'] : 'maps';
			$minExecs	= (isset($_GET['minexecs'])) ? $_GET['minexecs'] : -1;
			$minExecsFilter = "";

			if($minExecs > 0) $minExecsFilter = "HAVING COUNT(*) > $minExecs";
			
			$filter_execs = DBUtils::getFilterExecs();

			$options = Utils::getFilterOptions($db);
			$paramOptions = array();
			foreach($options[$paramEval] as $option)
			{
				if($paramEval == 'id_cluster') $paramOptions[] = $option['name'];
				else if($paramEval == 'comp') $paramOptions[] = Utils::getCompressionName($option[$paramEval]);
				else if($paramEval == 'net') $paramOptions[] = Utils::getNetworkName($option[$paramEval]);
				else if($paramEval == 'disk') $paramOptions[] = Utils::getDisksName($option[$paramEval]);
				else $paramOptions[] = $option[$paramEval];
			}

			$benchOptions = $db->get_rows("SELECT DISTINCT bench FROM execs WHERE 1 $filter_execs $where_configs GROUP BY $paramEval, bench order by $paramEval");
						
			// get the result rows
			$query = "SELECT count(*) as count, $paramEval, e.id_exec, exec as conf, bench, ".
				"exe_time, avg(exe_time) avg_exe_time, min(exe_time) min_exe_time ".
				"from execs e WHERE 1 $filter_execs $where_configs".
				"GROUP BY $paramEval, bench $minExecsFilter order by bench,$paramEval";
			$rows = $db->get_rows ( $query );
			if (!$rows) throw new \Exception ( "No results for query!" );
	
			$categories = '';
			$arrayBenchs = array();
			foreach ( $paramOptions as $param )
			{
				$categories .= "'$param ".Utils::getParamevalUnit($paramEval)."',";
				foreach($benchOptions as $bench) $arrayBenchs[$bench['bench']][$param] = null;
			}

			$series = array();
			$bench = '';
			foreach($rows as $row)
			{
				if($paramEval == 'comp') $row[$paramEval] = Utils::getCompressionName($row['comp']);
				else if($paramEval == 'id_cluster') $row[$paramEval] = Utils::getClusterName($row[$paramEval],$db);
				else if($paramEval == 'net') $row[$paramEval] = Utils::getNetworkName($row['net']);
				else if($paramEval == 'disk') $row[$paramEval] = Utils::getDisksName($row['disk']);
				else if($paramEval == 'iofilebuf') $row[$paramEval] /= 1024;
				
				$arrayBenchs[$row['bench']][$row[$paramEval]]['y'] = round((int)$row['avg_exe_time'],2);
				$arrayBenchs[$row['bench']][$row[$paramEval]]['count'] = (int)$row['count'];
			}				

			// ----------------------------------------------------
			// Add predictions to the series
			// ----------------------------------------------------

			$jsonData = $jsonHeader = "[]";
			$instance = "";
			$arrayBenchs_pred = array();

			$current_model = "";
			if (array_key_exists('current_model',$_GET)) $current_model = $_GET['current_model'];

			// compose instance
			$instance = MLUtils::generateSimpleInstance($param_names, $params, true, $db);
			$model_info = MLUtils::generateModelInfo($param_names, $params, true, $db);
			$instances = MLUtils::generateInstances($param_names, $params, true, $db);

			// model for filling
			$possible_models = $possible_models_id = array();
			MLUtils::findMatchingModels($model_info, $possible_models, $possible_models_id, $db);

			if (!empty($possible_models_id))
			{
				if ($current_model != "") $model = $current_model;
				else $current_model = $model = $possible_models_id[0];

				$cache_filename = getcwd().'/cache/query/'.md5($instance.'-'.$model).'-ipred.csv';
				$tmp_file = getcwd().'/cache/query/'.md5($instance.'-'.$model).'.tmp';

				$is_cached = file_exists($cache_filename);
				$in_process = file_exists(getcwd().'/cache/query/'.md5($instance.'-'.$model).'.lock');
				$finished_process = file_exists(getcwd().'/cache/query/'.md5($instance.'-'.$model).'.ready');
		
				if (!$is_cached && !$in_process && !$finished_process)
				{
					// drop query
					$command = getcwd().'/resources/queue -c "( cd '.getcwd().'/cache/query ; ';
					$command = $command.'touch '.getcwd().'/cache/query/'.md5($instance.'-'.$model).'.lock ; ';
					$command = $command.'rm -f '.$tmp_file.' ';
					foreach ($instances as $inst)
					{
						$command = $command.'&& '.getcwd().'/resources/aloja_cli.r -m aloja_predict_instance -l '.$model.' -p inst_predict=\''.$inst.'\' -v | grep -v \'WARNING\' | grep -v \'Prediction\' >> '.$tmp_file.' ';
					}
					$command = $command.'&& touch  '.getcwd().'/cache/query/'.md5($instance.'-'.$model).'.ready; ';<commit_msg>Corrected problem of lock + queue on ml_parameval<commit_after><?php

namespace alojaweb\Controller;

use alojaweb\inc\HighCharts;
use alojaweb\inc\Utils;
use alojaweb\inc\DBUtils;
use alojaweb\inc\MLUtils;

class MLParamevalController extends AbstractController
{
	public function mlparamEvaluationAction()
	{
		$db = $this->container->getDBUtils ();
		$rows = $categories = $series = '';
		try {
			$configurations = array ();	// Useless here
			$where_configs = '';
			$concat_config = ""; 		// Useless here
			
			$params = array();
			$param_names = array('benchs','nets','disks','mapss','iosfs','replications','iofilebufs','comps','blk_sizes','id_clusters'); // Order is important
			foreach ($param_names as $p) { $params[$p] = Utils::read_params($p,$where_configs,$configurations,$concat_config); sort($params[$p]); }

			if (count($_GET) <= 1
			|| (count($_GET) == 2 && array_key_exists('parameval',$_GET))
			|| (count($_GET) == 2 && array_key_exists('current_model',$_GET)))
			{
				$params['benchs'] = array('terasort');
				$where_configs = ' AND bench IN ("terasort")';
				if (!isset($_GET['parameval']) || $_GET['parameval'] != 'net') $params['nets'] = array('ETH'); $where_configs .= ' AND net IN ("ETH")';
				if (!isset($_GET['parameval']) || $_GET['parameval'] != 'disk') $params['disks'] = array('HDD','SSD'); $where_configs .= ' AND disk IN ("HDD","SSD")';
				if (!isset($_GET['parameval']) || $_GET['parameval'] != 'iofilebuf') $params['iofilebufs'] = array('32768','65536','131072'); $where_configs .= ' AND iofilebuf IN ("32768","65536","131072")';
				if (!isset($_GET['parameval']) || $_GET['parameval'] != 'iofs') $params['iosfs'] = array('10'); $where_configs .= ' AND iosf IN ("10")';
				if (!isset($_GET['parameval']) || $_GET['parameval'] != 'comp') $params['comps'] = array('0'); $where_configs .= ' AND comp IN ("0")';
				if (!isset($_GET['parameval']) || $_GET['parameval'] != 'replication') $params['replications'] = array('1'); $where_configs .= ' AND replication IN ("1")';
			}

			$money		= Utils::read_params ( 'money', $where_configs, $configurations, $concat_config );
			$paramEval	= (isset($_GET['parameval']) && $_GET['parameval'] != '') ? $_GET['parameval'] : 'maps';
			$minExecs	= (isset($_GET['minexecs'])) ? $_GET['minexecs'] : -1;
			$minExecsFilter = "";

			if($minExecs > 0) $minExecsFilter = "HAVING COUNT(*) > $minExecs";
			
			$filter_execs = DBUtils::getFilterExecs();

			$options = Utils::getFilterOptions($db);
			$paramOptions = array();
			foreach($options[$paramEval] as $option)
			{
				if($paramEval == 'id_cluster') $paramOptions[] = $option['name'];
				else if($paramEval == 'comp') $paramOptions[] = Utils::getCompressionName($option[$paramEval]);
				else if($paramEval == 'net') $paramOptions[] = Utils::getNetworkName($option[$paramEval]);
				else if($paramEval == 'disk') $paramOptions[] = Utils::getDisksName($option[$paramEval]);
				else $paramOptions[] = $option[$paramEval];
			}

			$benchOptions = $db->get_rows("SELECT DISTINCT bench FROM execs WHERE 1 $filter_execs $where_configs GROUP BY $paramEval, bench order by $paramEval");
						
			// get the result rows
			$query = "SELECT count(*) as count, $paramEval, e.id_exec, exec as conf, bench, ".
				"exe_time, avg(exe_time) avg_exe_time, min(exe_time) min_exe_time ".
				"from execs e WHERE 1 $filter_execs $where_configs".
				"GROUP BY $paramEval, bench $minExecsFilter order by bench,$paramEval";
			$rows = $db->get_rows ( $query );
			if (!$rows) throw new \Exception ( "No results for query!" );
	
			$categories = '';
			$arrayBenchs = array();
			foreach ( $paramOptions as $param )
			{
				$categories .= "'$param ".Utils::getParamevalUnit($paramEval)."',";
				foreach($benchOptions as $bench) $arrayBenchs[$bench['bench']][$param] = null;
			}

			$series = array();
			$bench = '';
			foreach($rows as $row)
			{
				if($paramEval == 'comp') $row[$paramEval] = Utils::getCompressionName($row['comp']);
				else if($paramEval == 'id_cluster') $row[$paramEval] = Utils::getClusterName($row[$paramEval],$db);
				else if($paramEval == 'net') $row[$paramEval] = Utils::getNetworkName($row['net']);
				else if($paramEval == 'disk') $row[$paramEval] = Utils::getDisksName($row['disk']);
				else if($paramEval == 'iofilebuf') $row[$paramEval] /= 1024;
				
				$arrayBenchs[$row['bench']][$row[$paramEval]]['y'] = round((int)$row['avg_exe_time'],2);
				$arrayBenchs[$row['bench']][$row[$paramEval]]['count'] = (int)$row['count'];
			}				

			// ----------------------------------------------------
			// Add predictions to the series
			// ----------------------------------------------------

			$jsonData = $jsonHeader = "[]";
			$instance = "";
			$arrayBenchs_pred = array();

			$current_model = "";
			if (array_key_exists('current_model',$_GET)) $current_model = $_GET['current_model'];

			// compose instance
			$instance = MLUtils::generateSimpleInstance($param_names, $params, true, $db);
			$model_info = MLUtils::generateModelInfo($param_names, $params, true, $db);
			$instances = MLUtils::generateInstances($param_names, $params, true, $db);

			// model for filling
			$possible_models = $possible_models_id = array();
			MLUtils::findMatchingModels($model_info, $possible_models, $possible_models_id, $db);

			if (!empty($possible_models_id))
			{
				if ($current_model != "") $model = $current_model;
				else $current_model = $model = $possible_models_id[0];

				$cache_filename = getcwd().'/cache/query/'.md5($instance.'-'.$model).'-ipred.csv';
				$tmp_file = getcwd().'/cache/query/'.md5($instance.'-'.$model).'.tmp';

				$is_cached = file_exists($cache_filename);
				$in_process = file_exists(getcwd().'/cache/query/'.md5($instance.'-'.$model).'.lock');
				$finished_process = file_exists(getcwd().'/cache/query/'.md5($instance.'-'.$model).'.ready');
		
				if (!$is_cached && !$in_process && !$finished_process)
				{
					// drop query
					$command = 'cd '.getcwd().'/cache/query ; ';
					$command = $command.'touch '.getcwd().'/cache/query/'.md5($instance.'-'.$model).'.lock ; ';
					$command = $command.getcwd().'/resources/queue -c "( ';
					$command = $command.'rm -f '.$tmp_file.' ';
					foreach ($instances as $inst)
					{
						$command = $command.'&& '.getcwd().'/resources/aloja_cli.r -m aloja_predict_instance -l '.$model.' -p inst_predict=\''.$inst.'\' -v | grep -v \'WARNING\' | grep -v \'Prediction\' >> '.$tmp_file.' ';
					}
					$command = $command.'&& touch  '.getcwd().'/cache/query/'.md5($instance.'-'.$model).'.ready; ';
{'fraction_non_alphanumeric': 0.23033751057123086, 'fraction_numerical': 0.009072038133312832, 'mean_word_length': 4.209451341609932, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4930661', 'n_tokens_mistral': 5073, 'n_tokens_neox': 4400, 'n_words': 805}