<commit_before>#!/usr/bin/php
<?php

date_default_timezone_set("UTC");

function config() {
    return array(
        "fp_project"=>"114",//114 for dev //71 for prod
        "fp_user"=>"goc",
        "fp_pass"=>"<PASSWORD>",
        "data_host"=>"data-itb.goc",
        "data_db"=>"gocticket",
        "data_user"=>"ticket",
        "data_pass"=>"<PASSWORD>"
    );
}

function parse_fpstr($str)
{
    $str = str_replace("__u", "-", $str);
    $str = str_replace("__b", " ", $str);
    $str = str_replace("__f", "/", $str);
    $str = str_replace("__P", "(", $str);
    $str = str_replace("__p", ")", $str);
    return $str;
}

function load_fp($id, &$details) {
    $config = config();

    ///////////////////////////////////////////////////////////////////////////////////////////////////
    //load detail from footprints
    try {
        $client = new SoapClient(NULL, array(
            "location"=>"https://tick.globalnoc.iu.edu/MRcgi/MRWebServices.pl", "uri"=>"MRWebServices",
            "style"=>SOAP_RPC, "use" => SOAP_ENCODED)
        );
        
        $fp_details = $client->MRWebServices__getIssueDetails_goc($config["fp_user"],$config["fp_pass"],'',$config["fp_project"],$id);
        //var_dump($fp_details);

        $details["title"] = $fp_details->title;
        $details["status"] = parse_fpstr($fp_details->status);
        $details["priority"] = $fp_details->priority;//TODO - conver this to string
        $details["nextaction"] = $fp_details->ENG__bNext__bAction__bItem;
        
        //process assignee / ccs
        $assignees = explode(" ", $fp_details->assignees);
        $formatted_assignees = array();
        $formatted_ccs = array();
        foreach($assignees as $assignee) {
            $assignee = parse_fpstr($assignee);
            if(strpos($assignee, "CC:") === 0) {
                $formatted_ccs[] = substr($assignee, 3);
            } else {
                //TODO - convert to user friendly user names
                $formatted_assignees[] = $assignee;
            }
        }
        $details["assignees"] = $formatted_assignees;
        $details["ccs"] = $formatted_ccs;

        //process description
        $descs = array();
        foreach($fp_details->allDescriptions as $desc) {
            $time = $desc->stamp;//TODO - what should I do with thi?
            $descs[$time] = $desc->data;
        }
        $details["descriptions"] = $descs;

    } catch (SoapFault $exception) {
        print "ERROR! - Got a SOAP exception:<br>";
        echo $exception;             
    }
}

function load_meta($id, $db, &$details) {
    $config = config();
    $projid = $config["fp_project"];

    $result = mysql_query("SELECT * FROM metadata WHERE ticket_id = $id AND project_id = $projid");
    while($row = mysql_fetch_array($result)) {
        if($row["value"] === "null") continue;
        switch($row["key"]) {
        case "SUPPORTING_SC_ID": $details["SUPPORTING_SC_ID"] = (int)$row["value"]; break;
        case "ASSOCIATED_VO_ID": $details["ASSOCIATED_VO_ID"] = (int)$row["value"]; break;
        case "GGUS_TICKET_ID": $details["GGUS_TICKET_ID"] = (int)$row["value"]; break;
        case "ASSOCIATED_RG_ID": $details["ASSOCIATED_RG_ID"] = (int)$row["value"]; break;
        case "ASSOCIATED_R_ID": $details["ASSOCIATED_R_ID"] = (int)$row["value"]; break;
        case "SUBMITTER_NAME": $details["SUBMITTER_NAME"] = (int)$row["value"]; break;
        }
        /*
mysql> select distinct(`key`) from metadata;
+----------------------+
| key                  |
+----------------------+
| SUBMITTED_VIA        |
| SUBMITTER_NAME       |
| ASSOCIATED_R_ID      |
| ASSOCIATED_R_NAME    |
| ASSOCIATED_RG_ID     |
| ASSOCIATED_RG_NAME   |
| ASSOCIATED_VO_ID     |
| SUPPORTING_SC_ID     |
| SUBMITTER_DN         |
| GGUS_PROBLEM_TYPE    |
| GGUS_TICKET_ID       |
| GGUS_TICKET_TYPE     |
| SERVICE_ID           |
| SERVICENOW_TICKET_ID |
| SUPPORTING_SC_NAME   |
| ASSOCIATED_VO_NAME   |
+----------------------+
16 rows in set (0.27 sec)
        */
    }
}

function xmlentities($s)
{
    static $patterns = null;
    static $replacements = null;
    static $translation = null;

    //remove non ascii --
    $output = str_replace("\n","[NEWLINE]",$s);
    $output = preg_replace('/[^(\x20-\x7F)]*/','', $output);
    $output = str_replace("[NEWLINE]","\n",$output);
    return htmlentities($output);
}

//get list of ticket ids that are recently updated
//TODO - add ability specify "updated since" 
function list_updated($lastrun_timestamp) {
    $config = config();
    $ids = array();
    try {
        $recent_date = date("Y-m-d G:i:s", $lastrun_timestamp); //24 hours
        echo "loading tickets updated since $recent_date\n";
        $query = "select mrID from MASTER".$config["fp_project"]." WHERE mrupdatedate >= '$recent_date'";
        $client = new SoapClient(NULL, array(
            "location"=>"https://tick.globalnoc.iu.edu/MRcgi/MRWebServices.pl", "uri"=>"MRWebServices",
            "style"=>SOAP_RPC, "use" => SOAP_ENCODED)
        );
        $tickets = $client->MRWebServices__search_goc($config["fp_user"],$config["fp_pass"],'',$query);
        foreach($tickets as $ticket) {
            $ids[] = $ticket->mrid;
        }
        //print_r($ids);
    } catch (SoapFault $exception) {
        print "ERROR! - Got a SOAP exception:<br>";
        echo $exception;             
    }
    return $ids;
}

function connect_metadb() {
    $config = config();
    $db = mysql_connect($config["data_host"], $config["data_user"], $config["data_pass"]);
    mysql_select_db($config["data_db"], $db);
    return $db;
}
///////////////////////////////////////////////////////////////////////////////////////////////////
// Load ticket details

$db = connect_metadb();

$lastrun_file = "lastrun.txt";
if(file_exists($lastrun_file)) {
    $lastrun_timestamp = (int)file_get_contents($lastrun_file);
} else {
    $lastrun_timestamp = time() - 3600*24*365;//1 year?
}

$ids = list_updated($lastrun_timestamp);
//$ids = array(11912, 11913, 11915, 11926, 11971, 11976, 11977, 11987, 11995, 11997, 12003, 12015, 12026, 12031, 8817, 8889);
$total = count($ids);<commit_msg>updating to use local config<commit_after>#!/usr/bin/php
<?php

date_default_timezone_set("UTC");

$config = require("config.php");

function parse_fpstr($str)
{
    $str = str_replace("__u", "-", $str);
    $str = str_replace("__b", " ", $str);
    $str = str_replace("__f", "/", $str);
    $str = str_replace("__P", "(", $str);
    $str = str_replace("__p", ")", $str);
    return $str;
}

function load_fp($id, &$details) {
    global $config;

    ///////////////////////////////////////////////////////////////////////////////////////////////////
    //load detail from footprints
    try {
        $client = new SoapClient(NULL, array(
            "location"=>"https://tick.globalnoc.iu.edu/MRcgi/MRWebServices.pl", "uri"=>"MRWebServices",
            "style"=>SOAP_RPC, "use" => SOAP_ENCODED)
        );
        
        $fp_details = $client->MRWebServices__getIssueDetails_goc($config["fp_user"],$config["fp_pass"],'',$config["fp_project"],$id);
        //var_dump($fp_details);

        $details["title"] = $fp_details->title;
        $details["status"] = parse_fpstr($fp_details->status);
        $details["priority"] = $fp_details->priority;//TODO - conver this to string
        $details["nextaction"] = $fp_details->ENG__bNext__bAction__bItem;
        
        //process assignee / ccs
        $assignees = explode(" ", $fp_details->assignees);
        $formatted_assignees = array();
        $formatted_ccs = array();
        foreach($assignees as $assignee) {
            $assignee = parse_fpstr($assignee);
            if(strpos($assignee, "CC:") === 0) {
                $formatted_ccs[] = substr($assignee, 3);
            } else {
                //TODO - convert to user friendly user names
                $formatted_assignees[] = $assignee;
            }
        }
        $details["assignees"] = $formatted_assignees;
        $details["ccs"] = $formatted_ccs;

        //process description
        $descs = array();
        foreach($fp_details->allDescriptions as $desc) {
            $time = $desc->stamp;//TODO - what should I do with thi?
            $descs[$time] = $desc->data;
        }
        $details["descriptions"] = $descs;

    } catch (SoapFault $exception) {
        print "ERROR! - Got a SOAP exception:<br>";
        echo $exception;             
    }
}

function load_meta($id, $db, &$details) {
    global $config;
    $projid = $config["fp_project"];

    $result = mysql_query("SELECT * FROM metadata WHERE ticket_id = $id AND project_id = $projid");
    while($row = mysql_fetch_array($result)) {
        if($row["value"] === "null") continue;
        switch($row["key"]) {
        case "SUPPORTING_SC_ID": $details["SUPPORTING_SC_ID"] = (int)$row["value"]; break;
        case "ASSOCIATED_VO_ID": $details["ASSOCIATED_VO_ID"] = (int)$row["value"]; break;
        case "GGUS_TICKET_ID": $details["GGUS_TICKET_ID"] = (int)$row["value"]; break;
        case "ASSOCIATED_RG_ID": $details["ASSOCIATED_RG_ID"] = (int)$row["value"]; break;
        case "ASSOCIATED_R_ID": $details["ASSOCIATED_R_ID"] = (int)$row["value"]; break;
        case "SUBMITTER_NAME": $details["SUBMITTER_NAME"] = (int)$row["value"]; break;
        }
        /*
mysql> select distinct(`key`) from metadata;
+----------------------+
| key                  |
+----------------------+
| SUBMITTED_VIA        |
| SUBMITTER_NAME       |
| ASSOCIATED_R_ID      |
| ASSOCIATED_R_NAME    |
| ASSOCIATED_RG_ID     |
| ASSOCIATED_RG_NAME   |
| ASSOCIATED_VO_ID     |
| SUPPORTING_SC_ID     |
| SUBMITTER_DN         |
| GGUS_PROBLEM_TYPE    |
| GGUS_TICKET_ID       |
| GGUS_TICKET_TYPE     |
| SERVICE_ID           |
| SERVICENOW_TICKET_ID |
| SUPPORTING_SC_NAME   |
| ASSOCIATED_VO_NAME   |
+----------------------+
16 rows in set (0.27 sec)
        */
    }
}

function xmlentities($s)
{
    static $patterns = null;
    static $replacements = null;
    static $translation = null;

    //remove non ascii --
    $output = str_replace("\n","[NEWLINE]",$s);
    $output = preg_replace('/[^(\x20-\x7F)]*/','', $output);
    $output = str_replace("[NEWLINE]","\n",$output);
    return htmlentities($output);
}

//get list of ticket ids that are recently updated
//TODO - add ability specify "updated since" 
function list_updated($lastrun_timestamp) {
    global $config;
    $ids = array();
    try {
        $recent_date = date("Y-m-d G:i:s", $lastrun_timestamp); //24 hours
        echo "loading tickets updated since $recent_date\n";
        $query = "select mrID from MASTER".$config["fp_project"]." WHERE mrupdatedate >= '$recent_date'";
        $client = new SoapClient(NULL, array(
            "location"=>"https://tick.globalnoc.iu.edu/MRcgi/MRWebServices.pl", "uri"=>"MRWebServices",
            "style"=>SOAP_RPC, "use" => SOAP_ENCODED)
        );
        $tickets = $client->MRWebServices__search_goc($config["fp_user"],$config["fp_pass"],'',$query);
        foreach($tickets as $ticket) {
            $ids[] = $ticket->mrid;
        }
        //print_r($ids);
    } catch (SoapFault $exception) {
        print "ERROR! - Got a SOAP exception:<br>";
        echo $exception;             
    }
    return $ids;
}

function connect_metadb() {
    global $config;
    $db = mysql_connect($config["data_host"], $config["data_user"], $config["data_pass"]);
    mysql_select_db($config["data_db"], $db);
    return $db;
}
///////////////////////////////////////////////////////////////////////////////////////////////////
// Load ticket details

$db = connect_metadb();

$lastrun_file = "lastrun.txt";
if(file_exists($lastrun_file)) {
    $lastrun_timestamp = (int)file_get_contents($lastrun_file);
} else {
    $lastrun_timestamp = time() - 3600*24*365;//1 year?
}

$ids = list_updated($lastrun_timestamp);
//$ids = array(11912, 11913, 11915, 11926, 11971, 11976, 11977, 11987, 11995, 11997, 12003, 12015, 12026, 12031, 8817, 8889);
$total = count($ids);
{'fraction_non_alphanumeric': 0.20758333333333334, 'fraction_numerical': 0.017333333333333333, 'mean_word_length': 2.442627653471027, 'pattern_counts': {'":': 12, '<': 11, '<?xml version=': 0, '>': 58, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5191502', 'n_tokens_mistral': 4213, 'n_tokens_neox': 3960, 'n_words': 713}