<commit_before>#!/usr/bin/env python

import ConfigParser
import logging
import os
import pickle
import pipes
import re
import sys
import time
import traceback
from datetime import datetime
from logging.handlers import RotatingFileHandler

import ispyb

def get_logical_arg(job, name):
    return job.args[job.args.index(name)+1]

def format_time(t):
    return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(t))

def get_scaling_id(dir):
    '''Read the fast_dp output file, extract and return the autoProcScalingId'''
    id_file = os.path.join(dir, "ispyb_ids.xml")

    if (not os.path.isfile(id_file)) or (not os.access(id_file, os.R_OK)):
        print "Either file %s is missing or is not readable" % id_file
        return None
    xml = None
    with open(id_file, 'rb') as f:
        xml = f.read()
    if xml is None:
        return None

    m = re.match(r'.*\<autoProcScalingId\>(\d+)\<\/autoProcScalingId\>.*', xml)
    if m is None:
        return None
    else:
        return m.group(1)

def store_result(conn, dir, scaling_id):
    '''Store results from DIMPLE pipeline'''

    mx_processing = conn.mx_processing

    log_file = os.path.join(dir, "dimple.log")
    if (not os.path.isfile(log_file)) or (not os.access(log_file, os.R_OK)):
        print "Either file %s is missing or is not readable" % log_file
        return None

    log = ConfigParser.RawConfigParser()
    log.read(log_file)

    params = mx_processing.get_run_params()
    params['parentid'] = scaling_id
    params['pipeline'] = 'dimple'
    params['log_file'] = log_file
    params['success'] = 1

    starttime = log.get(log.sections()[1], 'start_time')
    params['starttime'] = datetime.strptime(starttime, '%Y-%m-%d %H:%M:%S')
    endtime = log.get(log.sections()[-1], 'end_time')
    params['endtime'] = datetime.strptime(endtime, '%Y-%m-%d %H:%M:%S')

    params['rfree_start'] = log.getfloat('refmac5 restr', 'ini_free_r')
    params['rfree_end'] = log.getfloat('refmac5 restr', 'free_r')

    params['r_start'] = log.getfloat('refmac5 restr', 'ini_overall_r')
    params['r_end'] = log.getfloat('refmac5 restr', 'overall_r')
    params['message'] = " ".join(log.get('find-blobs', 'info').split()[:4])
    params['run_dir'] = dir
    dimple_args = log.get('workflow', 'args').split()
    params['input_MTZ_file'] = dimple_args[0]
    params['input_coord_file'] = dimple_args[1]
    params['output_MTZ_file'] = dir + '/final.mtz'
    params['output_coord_file'] = dir + '/final.pdb'
    params['cmd_line'] = log.get('workflow', 'prog') + ' ' + log.get('workflow', 'args').replace('\n', ' ')
    mr_id = mx_processing.upsert_run(params.values())

    for n in (1,2):
        if os.path.exists(dir+'/blob{0}v1.png'.format(n)):
            blobparam = mx_processing.get_run_blob_params()
            blobparam['parentid'] = mr_id
            blobparam['view1'] = 'blob{0}v1.png'.format(n)
            blobparam['view2'] = 'blob{0}v2.png'.format(n)
            blobparam['view3'] = 'blob{0}v3.png'.format(n)
            mrblob_id = mx_processing.upsert_run_blob(blobparam.values())


def store_failure(conn, run_dir, scaling_id):
    '''Store failure of DIMPLE pipeline'''
    mx_processing = conn.mx_processing

    params = mx_processing.get_run_params()
    params['parentid'] = scaling_id
    params['pipeline'] = 'dimple'
    params['success'] = 0
    params['message'] = 'Unknown error'
    params['run_dir'] = run_dir
    mr_id = mx_processing.upsert_run(params.values())


# Configure logging
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter('* %(asctime)s [id=%(thread)d] <%(levelname)s> %(message)s')<commit_msg>Force ordered dictionary view into a list<commit_after>#!/usr/bin/env python

import ConfigParser
import logging
import os
import pickle
import pipes
import re
import sys
import time
import traceback
from datetime import datetime
from logging.handlers import RotatingFileHandler

import ispyb

def get_logical_arg(job, name):
    return job.args[job.args.index(name)+1]

def format_time(t):
    return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(t))

def get_scaling_id(dir):
    '''Read the fast_dp output file, extract and return the autoProcScalingId'''
    id_file = os.path.join(dir, "ispyb_ids.xml")

    if (not os.path.isfile(id_file)) or (not os.access(id_file, os.R_OK)):
        print "Either file %s is missing or is not readable" % id_file
        return None
    xml = None
    with open(id_file, 'rb') as f:
        xml = f.read()
    if xml is None:
        return None

    m = re.match(r'.*\<autoProcScalingId\>(\d+)\<\/autoProcScalingId\>.*', xml)
    if m is None:
        return None
    else:
        return m.group(1)

def store_result(conn, dir, scaling_id):
    '''Store results from DIMPLE pipeline'''

    mx_processing = conn.mx_processing

    log_file = os.path.join(dir, "dimple.log")
    if (not os.path.isfile(log_file)) or (not os.access(log_file, os.R_OK)):
        print "Either file %s is missing or is not readable" % log_file
        return None

    log = ConfigParser.RawConfigParser()
    log.read(log_file)

    params = mx_processing.get_run_params()
    params['parentid'] = scaling_id
    params['pipeline'] = 'dimple'
    params['log_file'] = log_file
    params['success'] = 1

    starttime = log.get(log.sections()[1], 'start_time')
    params['starttime'] = datetime.strptime(starttime, '%Y-%m-%d %H:%M:%S')
    endtime = log.get(log.sections()[-1], 'end_time')
    params['endtime'] = datetime.strptime(endtime, '%Y-%m-%d %H:%M:%S')

    params['rfree_start'] = log.getfloat('refmac5 restr', 'ini_free_r')
    params['rfree_end'] = log.getfloat('refmac5 restr', 'free_r')

    params['r_start'] = log.getfloat('refmac5 restr', 'ini_overall_r')
    params['r_end'] = log.getfloat('refmac5 restr', 'overall_r')
    params['message'] = " ".join(log.get('find-blobs', 'info').split()[:4])
    params['run_dir'] = dir
    dimple_args = log.get('workflow', 'args').split()
    params['input_MTZ_file'] = dimple_args[0]
    params['input_coord_file'] = dimple_args[1]
    params['output_MTZ_file'] = dir + '/final.mtz'
    params['output_coord_file'] = dir + '/final.pdb'
    params['cmd_line'] = log.get('workflow', 'prog') + ' ' + log.get('workflow', 'args').replace('\n', ' ')
    mr_id = mx_processing.upsert_run(list(params.values()))

    for n in (1,2):
        if os.path.exists(dir+'/blob{0}v1.png'.format(n)):
            blobparam = mx_processing.get_run_blob_params()
            blobparam['parentid'] = mr_id
            blobparam['view1'] = 'blob{0}v1.png'.format(n)
            blobparam['view2'] = 'blob{0}v2.png'.format(n)
            blobparam['view3'] = 'blob{0}v3.png'.format(n)
            mrblob_id = mx_processing.upsert_run_blob(list(blobparam.values()))


def store_failure(conn, run_dir, scaling_id):
    '''Store failure of DIMPLE pipeline'''
    mx_processing = conn.mx_processing

    params = mx_processing.get_run_params()
    params['parentid'] = scaling_id
    params['pipeline'] = 'dimple'
    params['success'] = 0
    params['message'] = 'Unknown error'
    params['run_dir'] = run_dir
    mr_id = mx_processing.upsert_run(list(params.values()))


# Configure logging
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter('* %(asctime)s [id=%(thread)d] <%(levelname)s> %(message)s')
{'fraction_non_alphanumeric': 0.16132536664856056, 'fraction_numerical': 0.007061379684953829, 'mean_word_length': 3.7546804389928985, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5352543', 'n_tokens_mistral': 2621, 'n_tokens_neox': 2582, 'n_words': 525}