<commit_before><?php

    use Stomp\Stomp;
    use Stomp\Message\Map;


    class Process extends Page {
        
        public static $arg_list = array(
            'PROCESSINGJOBID' => '\d+',
            'DATACOLLECTIONID' => '\d+',
            'DISPLAYNAME' => '([\w\s-])+',
            'COMMENTS' => '.*',

            'PROCESSINGJOBIMAGESWEEPID' => '\d+',
            'STARTIMAGE' => '\d+',
            'ENDIMAGE' => '\d+',
            
            'PROCESSINGJOBPARAMETERID' => '\d+',
            'PARAMETERKEY' => '\w+',
            'PARAMETERVALUE' => '([\w-\.,])+',

            'RECIPE' => '([\w-])+',
        );
        

        public static $dispatch = array(
            array('(/:PROCESSINGJOBID)', 'get', '_get_reprocessing'),
            array('', 'post', '_add_reprocessing'),

            array('/params(/:PROCESSINGJOBPARAMETERID)', 'get', '_get_reprocessing_params'),
            array('/params', 'post', '_add_reprocessing_params'),

            array('/sweeps(/:PROCESSINGJOBIMAGESWEEPID)', 'get', '_get_reprocessing_sweeps'),
            array('/sweeps', 'post', '_add_reprocessing_sweeps'),

            array('/enqueue', 'post', '_enqueue'),
        );


        # Reprocessings
        function _get_reprocessing() {
            if (!$this->has_arg('prop')) $this->_error('No proposal specified');

            $where = 'p.proposalid=:1';
            $args = array($this->proposalid);

            if ($this->has_arg('PROCESSINGJOBID')) {
                $where .= ' AND rp.processingjobid=:'.(sizeof($args)+1);
                array_push($args, $this->arg('PROCESSINGJOBID'));
            }

            if ($this->has_arg('VISIT')) {
                $where .= " AND CONCAT(p.proposalcode,p.proposalnumber,'-',s.visit_number) LIKE :".(sizeof($args)+1);
                array_push($args, $this->arg('VISIT'));
            }


            $tot = $this->db->pq("SELECT count(distinct rp.processingjobid) as tot 
                FROM processingjob rp
                INNER JOIN datacollection dc ON dc.datacollectionid = rp.datacollectionid
                INNER JOIN blsession s ON s.sessionid = dc.sessionid
                INNER JOIN proposal p ON p.proposalid = s.proposalid
                WHERE $where", $args);
            $tot = intval($tot[0]['TOT']);
            
            $start = 0;
            $pp = $this->has_arg('per_page') ? $this->arg('per_page') : 15;
            $end = $pp;
            
            if ($this->has_arg('page')) {
                $pg = $this->arg('page') - 1;
                $start = $pg*$pp;
                $end = $pg*$pp+$pp;
            }
            
            $st = sizeof($args)+1;
            array_push($args, $start);
            array_push($args, $end);
            

            $rows = $this->db->paginate("SELECT
                rp.processingjobid, rp.displayname, rp.comments, TO_CHAR(rp.recordtimestamp, 'DD-MM-YYYY HH24:MI') as recordtimestamp, rp.automatic, 
                dc.filetemplate, dc.imagedirectory, dc.datacollectionid, dc.imageprefix, dc.datacollectionnumber,
                smp.name as sample, smp.blsampleid as blsampleid, pr.acronym as protein, pr.proteinid,
                CONCAT(p.proposalcode,p.proposalnumber,'-',s.visit_number) as visit,

                apss.cchalf, apss.ccanomalous, apss.anomalous, apss.ntotalobservations as ntobs, apss.ntotaluniqueobservations as nuobs, apss.resolutionlimitlow as rlow, apss.resolutionlimithigh as rhigh, apss.rmeasalliplusiminus as rmeas, apss.rmerge, apss.completeness, apss.anomalouscompleteness as anomcompleteness, apss.anomalousmultiplicity as anommultiplicity, apss.multiplicity, apss.meanioversigi as isigi,
                IF(app.autoprocprogramid, IF(api.autoprocintegrationid, 1, app.processingstatus), 2) as status, app.processingmessage, TO_CHAR(app.processingendtime, 'DD-MM-YYYY HH24:MI') as lastupdatetimestamp
                FROM processingjob rp
                INNER JOIN datacollection dc ON dc.datacollectionid = rp.datacollectionid
                LEFT OUTER JOIN blsample smp ON smp.blsampleid = dc.blsampleid
                LEFT OUTER JOIN crystal cr ON cr.crystalid = smp.crystalid
                LEFT OUTER JOIN protein pr ON pr.proteinid = cr.proteinid
                INNER JOIN blsession s ON s.sessionid = dc.sessionid
                INNER JOIN proposal p ON p.proposalid = s.proposalid

                LEFT OUTER JOIN autoprocprogram app ON app.processingjobid = rp.processingjobid
                LEFT OUTER JOIN autoprocintegration api ON api.autoprocprogramid = app.autoprocprogramid
                LEFT OUTER JOIN autoprocscaling_has_int aph ON api.autoprocintegrationid = aph.autoprocintegrationid 
                LEFT OUTER JOIN autoprocscaling aps ON aph.autoprocscalingid = aps.autoprocscalingid 
                LEFT OUTER JOIN autoprocscalingstatistics apss ON (apss.autoprocscalingid = aph.autoprocscalingid AND scalingstatisticstype = 'overall')

                WHERE $where
                GROUP BY rp.processingjobid
                ORDER BY rp.processingjobid DESC", $args);    

            foreach ($rows as &$r) {
                $r['IMAGEDIRECTORYFULL'] = $r['IMAGEDIRECTORY'];
                $r['IMAGEDIRECTORY'] = preg_replace('/(.*?\/'.$r['VISIT'].')/', '', $r['IMAGEDIRECTORY'], 1);
            }
<commit_msg>sort out status for failed reprocessing jobs<commit_after><?php

    use Stomp\Stomp;
    use Stomp\Message\Map;


    class Process extends Page {
        
        public static $arg_list = array(
            'PROCESSINGJOBID' => '\d+',
            'DATACOLLECTIONID' => '\d+',
            'DISPLAYNAME' => '([\w\s-])+',
            'COMMENTS' => '.*',

            'PROCESSINGJOBIMAGESWEEPID' => '\d+',
            'STARTIMAGE' => '\d+',
            'ENDIMAGE' => '\d+',
            
            'PROCESSINGJOBPARAMETERID' => '\d+',
            'PARAMETERKEY' => '\w+',
            'PARAMETERVALUE' => '([\w-\.,])+',

            'RECIPE' => '([\w-])+',
        );
        

        public static $dispatch = array(
            array('(/:PROCESSINGJOBID)', 'get', '_get_reprocessing'),
            array('', 'post', '_add_reprocessing'),

            array('/params(/:PROCESSINGJOBPARAMETERID)', 'get', '_get_reprocessing_params'),
            array('/params', 'post', '_add_reprocessing_params'),

            array('/sweeps(/:PROCESSINGJOBIMAGESWEEPID)', 'get', '_get_reprocessing_sweeps'),
            array('/sweeps', 'post', '_add_reprocessing_sweeps'),

            array('/enqueue', 'post', '_enqueue'),
        );


        # Reprocessings
        function _get_reprocessing() {
            if (!$this->has_arg('prop')) $this->_error('No proposal specified');

            $where = 'p.proposalid=:1';
            $args = array($this->proposalid);

            if ($this->has_arg('PROCESSINGJOBID')) {
                $where .= ' AND rp.processingjobid=:'.(sizeof($args)+1);
                array_push($args, $this->arg('PROCESSINGJOBID'));
            }

            if ($this->has_arg('VISIT')) {
                $where .= " AND CONCAT(p.proposalcode,p.proposalnumber,'-',s.visit_number) LIKE :".(sizeof($args)+1);
                array_push($args, $this->arg('VISIT'));
            }


            $tot = $this->db->pq("SELECT count(distinct rp.processingjobid) as tot 
                FROM processingjob rp
                INNER JOIN datacollection dc ON dc.datacollectionid = rp.datacollectionid
                INNER JOIN blsession s ON s.sessionid = dc.sessionid
                INNER JOIN proposal p ON p.proposalid = s.proposalid
                WHERE $where", $args);
            $tot = intval($tot[0]['TOT']);
            
            $start = 0;
            $pp = $this->has_arg('per_page') ? $this->arg('per_page') : 15;
            $end = $pp;
            
            if ($this->has_arg('page')) {
                $pg = $this->arg('page') - 1;
                $start = $pg*$pp;
                $end = $pg*$pp+$pp;
            }
            
            $st = sizeof($args)+1;
            array_push($args, $start);
            array_push($args, $end);
            

            $rows = $this->db->paginate("SELECT
                rp.processingjobid, rp.displayname, rp.comments, TO_CHAR(rp.recordtimestamp, 'DD-MM-YYYY HH24:MI') as recordtimestamp, rp.automatic, 
                dc.filetemplate, dc.imagedirectory, dc.datacollectionid, dc.imageprefix, dc.datacollectionnumber,
                smp.name as sample, smp.blsampleid as blsampleid, pr.acronym as protein, pr.proteinid,
                CONCAT(p.proposalcode,p.proposalnumber,'-',s.visit_number) as visit,

                apss.cchalf, apss.ccanomalous, apss.anomalous, apss.ntotalobservations as ntobs, apss.ntotaluniqueobservations as nuobs, apss.resolutionlimitlow as rlow, apss.resolutionlimithigh as rhigh, apss.rmeasalliplusiminus as rmeas, apss.rmerge, apss.completeness, apss.anomalouscompleteness as anomcompleteness, apss.anomalousmultiplicity as anommultiplicity, apss.multiplicity, apss.meanioversigi as isigi,
                IF(app.autoprocprogramid, IF(api.autoprocintegrationid IS NOT NULL AND app.processingstatus = 1, 1, app.processingstatus), 2) as status, app.processingmessage, TO_CHAR(app.processingendtime, 'DD-MM-YYYY HH24:MI') as lastupdatetimestamp
                FROM processingjob rp
                INNER JOIN datacollection dc ON dc.datacollectionid = rp.datacollectionid
                LEFT OUTER JOIN blsample smp ON smp.blsampleid = dc.blsampleid
                LEFT OUTER JOIN crystal cr ON cr.crystalid = smp.crystalid
                LEFT OUTER JOIN protein pr ON pr.proteinid = cr.proteinid
                INNER JOIN blsession s ON s.sessionid = dc.sessionid
                INNER JOIN proposal p ON p.proposalid = s.proposalid

                LEFT OUTER JOIN autoprocprogram app ON app.processingjobid = rp.processingjobid
                LEFT OUTER JOIN autoprocintegration api ON api.autoprocprogramid = app.autoprocprogramid
                LEFT OUTER JOIN autoprocscaling_has_int aph ON api.autoprocintegrationid = aph.autoprocintegrationid 
                LEFT OUTER JOIN autoprocscaling aps ON aph.autoprocscalingid = aps.autoprocscalingid 
                LEFT OUTER JOIN autoprocscalingstatistics apss ON (apss.autoprocscalingid = aph.autoprocscalingid AND scalingstatisticstype = 'overall')

                WHERE $where
                GROUP BY rp.processingjobid
                ORDER BY rp.processingjobid DESC", $args);    

            foreach ($rows as &$r) {
                $r['IMAGEDIRECTORYFULL'] = $r['IMAGEDIRECTORY'];
                $r['IMAGEDIRECTORY'] = preg_replace('/(.*?\/'.$r['VISIT'].')/', '', $r['IMAGEDIRECTORY'], 1);
            }

{'fraction_non_alphanumeric': 0.12901418969380135, 'fraction_numerical': 0.0030806572068707992, 'mean_word_length': 2.11333914559721, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2215086', 'n_tokens_mistral': 3531, 'n_tokens_neox': 3283, 'n_words': 668}