<commit_before><?

require_once("app/timerange.php");

//This class takes care of all cron-ish jobs that are initiated from cron on localhost
//wget -O latestmetric_log http://rsv-itb.grid.iu.edu/trunk/cron/latestmetric
class CronController extends Zend_Controller_Action 
{ 
    public function init()
    {
        //make sure the request originated from localhost
        if(!config()->debug) {
            if($_SERVER["REMOTE_ADDR"] != $_SERVER["SERVER_ADDR"]) {
                //pretend that this page doesn't exist
                $this->getResponse()->setRawHeader('HTTP/1.1 404 Not Found');
                echo "404";
                exit;
            }
        }
    }

    //load new record from Gratia db and load it to our DB tables
    public function processnewAction()
    {
        dlog("processing new records from gratia table.");

        //why flock? sometimes processnew takes very long time to complete, and if the jobs is kicking off 
        //every minute, the overwrap could happen. If it does, skip it.
        $fp_lock = fopen("/tmp/dashboard.processnew", "w+");
        dlog("acquiring flock.");
        if (flock($fp_lock, LOCK_EX | LOCK_NB)) {
            dlog("acquiring flock success...");

            $resource_model = new Resource();
            $probe_model = new ProbeInfo();
            $metric_model = new Metrics();

            //grab sets of latest metrics for each resources for status caluculations
            $current_metrics = array();
            $overall_status_model = array();
            $overall_status = array();

            //for some status..
            $rejected = 0;
            $inserted = 0;
            $validation_error = 0;
            $newstatus_inserted = 0;

            //grab "some" new records
            $newrecords = $metric_model->fetchNewGratiaRecords(5000);
            dlog("Records grabbed from gratia: ".count($newrecords));

            //we are going to make repeated inserts (x thousands times). Let's disable profiling for now
            Zend_Registry::get('db')->getProfiler()->setEnabled(false);

            require_once("app/plugins/metricupdate.php");
            $plugin = new Plugin_MetricUpdate();

            foreach($newrecords as $record) {
                //pull & gather information about this metric data
                $dbid = $record->dbid;
                $resource_id = $resource_model->lookupID($record->ServiceUri);
                $metric_id = $probe_model->lookupID($record->MetricName);
                $status = $record->MetricStatus;
                $timestamp = $record->unix_timestamp;
                $detail = $record->DetailsData;
                $effective_dbid = null;
                $effective_timestamp = null;

                //validate the metrci data
                if($resource_id === null) {
                    $validation_error++;
                    continue;
                }
                if($metric_id === null) {
                    $validation_error++;
                    continue;
                }

                //pull current metrics for this resource (for initial metric set)
                $current = array();
                if(!isset($current_metrics[$resource_id])) {
                    dlog("initializing current_metrics array for $resource_id");

                    //first time..
                    $current = $metric_model->getLatest($resource_id);

                    //find the last metric entered and use it as resource timestamp (to detect out-of-order metric data)
                    $last = 0;
                    foreach($current as $met) {
                        if($met->timestamp > $last) $last = $met->timestamp;
                    }
                    $current["lasttime"] = $last;
                    //dlog("last metric entry for $resource_id happend at ".date(config()->date_format_full." s", $last));

                    $current_metrics[$resource_id] = $current;
                }
                $current = $current_metrics[$resource_id];

                //if status is unknown, set effective_dbic to last known metric if it is 
                //within metric_considered_old time.
                if($status == "UNKNOWN") {
                    if(isset($current[$metric_id])) {
                        $previous = $current[$metric_id];
                        if($previous->status == "UNKNOWN") {
                            //previous is also UNKNOWN, test with it's effective_ metric info
                            if(($previous->effective_timestamp + config()->metric_considered_old) > $timestamp) {
                                $effective_dbid = $previous->effective_dbid;
                                $effective_timestamp = $previous->effective_timestamp;
                            }
                        } else {
                            //previous is not UNNOWN, test with it's status
                            if(($previous->timestamp + config()->metric_considered_old) > $timestamp) {
                                $effective_dbid = $previous->dbid;
                                $effective_timestamp = $previous->timestamp;
                            }
                        }
                        if($effective_dbid !== null) {
                            dlog("setting effective status on metric ID: $dbid with that of metric ID: $effective_dbid");
                        }
                    }
                }

                //update the current set with the new record from gratia
                if(!isset($current[$metric_id])) {
                    //first ever metric type for this resource!
                    $current[$metric_id] = new MetricRecord(); //metric record is defined in config.php (for now)
                }
                $current[$metric_id]->dbid = $dbid;
                $current[$metric_id]->status = $status;
                $current[$metric_id]->timestamp = $timestamp;
                $current[$metric_id]->detail = $detail;
                $current[$metric_id]->effective_dbid = $effective_dbid;
                $current[$metric_id]->effective_timestamp = $effective_timestamp;

                //check the lasttimestamp to detect the out-of-order metric -- 
                //(TODO: Arvind to contact Gratia to see if this could be prevented upstream)
                if($current["lasttime"] > $timestamp) {
                    $lag = $current["lasttime"] - $timestamp;
                    if($lag < config()->gratia_max_outoforder) {
                        elog("out-of-order metric detected. ID:$dbid ".$lag. " seconds - resetting timestamp to last reported timestamp");
                        $timestamp = $current["lasttime"];
                    } else {
                        elog("out-of-order metric detected. ID:$dbid ".$lag. " seconds (more than ".config()->gratia_max_outoforder." seconds) - ignoring");
                        //TODO - Arvind wants to mark these records, and do post-processing. I think if we have to develop
                        //2 sets of algorithms to handle this, then I should get this algorithm smart enough to be able
                        //to handle the late records.
                        continue;
                    }
                }
                $current["lasttime"] = $timestamp;

                //insert to our metrics table
                try {
                    $metric_model->insert($dbid, $resource_id, $metric_id, $status, $timestamp, $detail, $effective_dbid, $effective_timestamp);
                    //dlog("inserting new metric $dbid for resource: $resource_id at ".date(config()->date_format_full." s", $timestamp));
                    $inserted++;
                } catch (Exception $e) {
                    $rejected++;
                    elog("Caught Exception while running query: ".print_r($e, true));
                }

                //handle metric update plugins
                $plugin->dispatch($resource_id, $metric_id, $current[$metric_id]);

                //re-calculate overall status
                if(!isset($overall_status[$resource_id])) {
                    //we have not yet seen this resource id appears initialize aux. classes.
                    dlog("initializing overallstatus model for $resource_id");
                    $ostatus_model = new OverallStatus($resource_id);
                    $overall_status_model[$resource_id] = $ostatus_model;
                    $lastinfo = $ostatus_model->getLastInfo();
                    if(isset($lastinfo->overall_status)) {
                        $overall_status[$resource_id] = $lastinfo->overall_status;
                    } else {
                        //below if ($new_status != overall_status) will cause this record to be inserted as initial status.
                        $overall_status[$resource_id] = "no-last-info"; 
                    }
                } else {
                    $ostatus_model = $overall_status_model[$resource_id];
                }
                $ostatus_model->calculateStatus($current, $timestamp);
                $new_status = $ostatus_model->getOverallStatus();

                //insert to status change table if ostatus changed
                if($overall_status[$resource_id] != $new_status) {

                    //if status changed due to metric being expired, then use the expiration time
                    //and its metric id for this status change.
                    //I can then ignore the metric itself that triggered the status to change to UNKNOWN
                    //since the status will remain unknown.
                    if($ostatus_model->isExpired()) {<commit_msg>made resource model initialize at the same time current_metric iniailize<commit_after><?

require_once("app/timerange.php");

//This class takes care of all cron-ish jobs that are initiated from cron on localhost
//wget -O latestmetric_log http://rsv-itb.grid.iu.edu/trunk/cron/latestmetric
class CronController extends Zend_Controller_Action 
{ 
    public function init()
    {
        //make sure the request originated from localhost
        if(!config()->debug) {
            if($_SERVER["REMOTE_ADDR"] != $_SERVER["SERVER_ADDR"]) {
                //pretend that this page doesn't exist
                $this->getResponse()->setRawHeader('HTTP/1.1 404 Not Found');
                echo "404";
                exit;
            }
        }
    }

    //load new record from Gratia db and load it to our DB tables
    public function processnewAction()
    {
        dlog("processing new records from gratia table.");

        //why flock? sometimes processnew takes very long time to complete, and if the jobs is kicking off 
        //every minute, the overwrap could happen. If it does, skip it.
        $fp_lock = fopen("/tmp/dashboard.processnew", "w+");
        dlog("acquiring flock.");
        if (flock($fp_lock, LOCK_EX | LOCK_NB)) {
            dlog("acquiring flock success...");

            $resource_model = new Resource();
            $probe_model = new ProbeInfo();
            $metric_model = new Metrics();

            //grab sets of latest metrics for each resources for status caluculations
            $current_metrics = array();
            $overall_status_model = array();
            $overall_status = array();

            //for some status..
            $rejected = 0;
            $inserted = 0;
            $validation_error = 0;
            $newstatus_inserted = 0;

            //grab "some" new records
            $newrecords = $metric_model->fetchNewGratiaRecords(5000);
            dlog("Records grabbed from gratia: ".count($newrecords));

            //we are going to make repeated inserts (x thousands times). Let's disable profiling for now
            Zend_Registry::get('db')->getProfiler()->setEnabled(false);

            require_once("app/plugins/metricupdate.php");
            $plugin = new Plugin_MetricUpdate();

            foreach($newrecords as $record) {
                //pull & gather information about this metric data
                $dbid = $record->dbid;
                $resource_id = $resource_model->lookupID($record->ServiceUri);
                $metric_id = $probe_model->lookupID($record->MetricName);
                $status = $record->MetricStatus;
                $timestamp = $record->unix_timestamp;
                $detail = $record->DetailsData;
                $effective_dbid = null;
                $effective_timestamp = null;

                //validate the metrci data
                if($resource_id === null) {
                    $validation_error++;
                    continue;
                }
                if($metric_id === null) {
                    $validation_error++;
                    continue;
                }

                //pull current metrics for this resource (for initial metric set)
                $current = array();
                if(!isset($current_metrics[$resource_id])) {
                    dlog("initializing current_metrics array for $resource_id");
                    $current = $metric_model->getLatest($resource_id);

                    //find the last metric entered and use it as resource timestamp (to detect out-of-order metric data)
                    $last = 0;
                    foreach($current as $met) {
                        if($met->timestamp > $last) $last = $met->timestamp;
                    }
                    $current["lasttime"] = $last;
                    //dlog("last metric entry for $resource_id happend at ".date(config()->date_format_full." s", $last));

                    $current_metrics[$resource_id] = $current;

                    $ostatus_model = new OverallStatus($resource_id);
                    $overall_status_model[$resource_id] = $ostatus_model;
                    $lastinfo = $ostatus_model->getLastInfo();
                    if(isset($lastinfo->overall_status)) {
                        $overall_status[$resource_id] = $lastinfo->overall_status;
                    } else {
                        //below if ($new_status != overall_status) will cause this record to be inserted as initial status.
                        $overall_status[$resource_id] = "no-last-info"; 
                    }
                }
                $ostatus_model = $overall_status_model[$resource_id];
                $current = $current_metrics[$resource_id];
                
                //if status is unknown, set effective_dbic to last known metric if it is 
                //within metric_considered_old time.
                if($status == "UNKNOWN") {
                    if(isset($current[$metric_id])) {
                        $previous = $current[$metric_id];
                        if($previous->status == "UNKNOWN") {
                            //previous is also UNKNOWN, test with it's effective_ metric info
                            if(($previous->effective_timestamp + config()->metric_considered_old) > $timestamp) {
                                $effective_dbid = $previous->effective_dbid;
                                $effective_timestamp = $previous->effective_timestamp;
                            }
                        } else {
                            //previous is not UNNOWN, test with it's status
                            if(($previous->timestamp + config()->metric_considered_old) > $timestamp) {
                                $effective_dbid = $previous->dbid;
                                $effective_timestamp = $previous->timestamp;
                            }
                        }
                        if($effective_dbid !== null) {
                            dlog("setting effective status on metric ID: $dbid with that of metric ID: $effective_dbid");
                        }
                    }
                }

                //update the current set with the new record from gratia
                if(!isset($current[$metric_id])) {
                    //first ever metric type for this resource!
                    $current[$metric_id] = new MetricRecord(); //metric record is defined in config.php (for now)
                }
                $current[$metric_id]->dbid = $dbid;
                $current[$metric_id]->status = $status;
                $current[$metric_id]->timestamp = $timestamp;
                $current[$metric_id]->detail = $detail;
                $current[$metric_id]->effective_dbid = $effective_dbid;
                $current[$metric_id]->effective_timestamp = $effective_timestamp;

                //check the lasttimestamp to detect the out-of-order metric -- 
                //(TODO: Arvind to contact Gratia to see if this could be prevented upstream)
                if($current["lasttime"] > $timestamp) {
                    $lag = $current["lasttime"] - $timestamp;
                    if($lag < config()->gratia_max_outoforder) {
                        elog("out-of-order metric detected. ID:$dbid ".$lag. " seconds - resetting timestamp to last reported timestamp");
                        $timestamp = $current["lasttime"];
                    } else {
                        elog("out-of-order metric detected. ID:$dbid ".$lag. " seconds (more than ".config()->gratia_max_outoforder." seconds) - ignoring");
                        //TODO - Arvind wants to mark these records, and do post-processing. I think if we have to develop
                        //2 sets of algorithms to handle this, then I should get this algorithm smart enough to be able
                        //to handle the late records.
                        continue;
                    }
                }
                $current["lasttime"] = $timestamp;

                //insert to our metrics table
                try {
                    $metric_model->insert($dbid, $resource_id, $metric_id, $status, $timestamp, $detail, $effective_dbid, $effective_timestamp);
                    //dlog("inserting new metric $dbid for resource: $resource_id at ".date(config()->date_format_full." s", $timestamp));
                    $inserted++;
                } catch (Exception $e) {
                    $rejected++;
                    elog("Caught Exception while running query: ".print_r($e, true));
                }

                //handle metric update plugins
                $plugin->dispatch($resource_id, $metric_id, $current[$metric_id]);

                //re-calculate overall status
                $ostatus_model->calculateStatus($current, $timestamp);
                $new_status = $ostatus_model->getOverallStatus();

                //insert to status change table if ostatus changed
                if($overall_status[$resource_id] != $new_status) {

                    //if status changed due to metric being expired, then use the expiration time
                    //and its metric id for this status change.
                    //I can then ignore the metric itself that triggered the status to change to UNKNOWN
                    //since the status will remain unknown.
                    if($ostatus_model->isExpired()) {
{'fraction_non_alphanumeric': 0.10233167408959916, 'fraction_numerical': 0.0018862981399004454, 'mean_word_length': 1.5498997995991983, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 99, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7224981', 'n_tokens_mistral': 4779, 'n_tokens_neox': 4407, 'n_words': 1274}