<commit_before>#!/usr/bin/env python
# mxdatareduction2ispyb.py
#
#    Copyright (C) 2014 Diamond Light Source, <NAME>
#    
# 2015-01-05 
#
# Script to store e.g. xia2 and fast_dp results using the ispyb_api.  
#

import sys
from datetime import datetime
from xml.etree import ElementTree
from ispyb_api.dbconnection import dbconnection
from ispyb_api.core import core
from ispyb_api.mxdatareduction import mxdatareduction

# XML-to-dict code from here: 
# http://code.activestate.com/recipes/410469-xml-as-dictionary/

class XmlListConfig(list):
    def __init__(self, aList):
        for element in aList:
            if element:
                # treat like dict
                if len(element) == 1 or element[0].tag != element[1].tag:
                    self.append(XmlDictConfig(element))
                # treat like list
                elif element[0].tag == element[1].tag:
                    self.append(XmlListConfig(element))
            elif element.text:
                text = element.text.strip()
                if text:
                    self.append(text)
            elif element.items():
                self.append(OrderedDict(element.items()))

class XmlDictConfig(dict):
    '''
    Example usage:

    >>> tree = ElementTree.parse('your_file.xml')
    >>> root = tree.getroot()
    >>> xmldict = XmlDictConfig(root)

    Or, if you want to use an XML string:

    >>> root = ElementTree.XML(xml_string)
    >>> xmldict = XmlDictConfig(root)

    And then use xmldict for what it is... a dict.
    '''
    def __init__(self, parent_element):
        childrenNames = []
        for child in parent_element.getchildren():
            childrenNames.append(child.tag)
        
        if parent_element.items():
            self.update(dict(parent_element.items()))
        for element in parent_element:
            if len(element):  # was: if element:
                # treat like dict - we assume that if the first two tags
                # in a series are different, then they are all different.
                if len(element) == 1 or element[0].tag != element[1].tag:
                    aDict = XmlDictConfig(element)
                # treat like list - we assume that if the first two tags
                # in a series are the same, then the rest are the same.
                else:
                    # here, we put the list in dictionary; the key is the
                    # tag name the list elements all share in common, and
                    # the value is the list itself 
                    aDict = {element[0].tag: XmlListConfig(element)}
                # if the tag has attributes, add those to the dict
                if element.items():
                    aDict.update(dict(element.items()))
                    
                if childrenNames.count(element.tag) > 1:
                    try:
                        currentValue = self[element.tag]
                        currentValue.append(aDict)
                        self.update({element.tag: currentValue})
                    except: #the first of its kind, an empty list must be created
                        self.update({element.tag: [aDict]}) #aDict is written in [], i.e. it will be a list

                else:
                    self.update({element.tag: aDict})
            # this assumes that if you've got an attribute in a tag,
            # you won't be having any text. This may or may not be a 
            # good idea -- time will tell. It works for the way we are
            # currently doing XML configuration files...
            elif element.items():
                self.update({element.tag: dict(element.items())})
            # finally, if there are no child tags and no attributes, extract
            # the text
            else:
                self.update({element.tag: element.text})

if len(sys.argv) not in (2,3):
    print("Usage:")
    print("%s xml_in_file" % sys.argv[0])
    print("%s xml_in_file xml_out_file" % sys.argv[0])
    sys.exit(1)

# Convert the XML to a dictionary
tree = ElementTree.parse(sys.argv[1])
root = tree.getroot()
xmldict = XmlDictConfig(root)

# Convenience pointers and sanity checks
image = xmldict['AutoProcScalingContainer']['AutoProcIntegrationContainer']['Image']
integration = xmldict['AutoProcScalingContainer']['AutoProcIntegrationContainer']['AutoProcIntegration']
proc = xmldict['AutoProc']
program = xmldict['AutoProcProgramContainer']['AutoProcProgram']
attachments = xmldict['AutoProcProgramContainer']['AutoProcProgramAttachment']
if isinstance(attachments, dict): # Make it a list regardless
    attachments = [attachments]
scaling = xmldict['AutoProcScalingContainer']['AutoProcScaling']

if proc == None:
    sys.exit("ERROR - please make sure the XML file contains an AutoProc element.")
if scaling == None:
    sys.exit("ERROR - please make sure the XML file contains an AutoProcScaling element.")
if integration == None:
    sys.exit("ERROR - please make sure the XML file contains an AutoProcIntegration element.")
if image == None or image['fileName'] == None or image['fileLocation'] == None:
    sys.exit("ERROR - please make sure the XML file contains an Image element with fileName and fileLocation elements.")

s = [None, None, None]
for i in xrange(0,3):
    stats = xmldict['AutoProcScalingContainer']['AutoProcScalingStatistics'][i]
    if stats['scalingStatisticsType'] == 'outerShell':
        s[0] = stats
    elif stats['scalingStatisticsType'] == 'innerShell':
        s[1] = stats
    elif stats['scalingStatisticsType'] == 'overall':
        s[2] = stats

if s[0] == None or s[1] == None or s[2] == None:
    sys.exit("ERROR - please make sure the XML file contains 3 AutoProcScalingStatistics elements.")

# Get a database cursor
cursor = dbconnection.connect_to_prod()

# Find the datacollection associated with this data reduction run

dc_id = core.retrieve_datacollection_id(cursor, image['fileName'], image['fileLocation'])

# Store results from XIA2 / MX data reduction pipelines
# ...first the program info 
params = mxdatareduction.get_program_params()
if 'processingPrograms' in program:<commit_msg>Graeme's changes to use .dc_id file for dataCollectionId<commit_after>#!/usr/bin/env python
# mxdatareduction2ispyb.py
#
#    Copyright (C) 2014 Diamond Light Source, <NAME>
#    
# 2015-01-05 
#
# Script to store e.g. xia2 and fast_dp results using the ispyb_api.  
#

import sys
import os
from datetime import datetime
from xml.etree import ElementTree
from ispyb_api.dbconnection import dbconnection
from ispyb_api.core import core
from ispyb_api.mxdatareduction import mxdatareduction

# XML-to-dict code from here: 
# http://code.activestate.com/recipes/410469-xml-as-dictionary/

class XmlListConfig(list):
    def __init__(self, aList):
        for element in aList:
            if element:
                # treat like dict
                if len(element) == 1 or element[0].tag != element[1].tag:
                    self.append(XmlDictConfig(element))
                # treat like list
                elif element[0].tag == element[1].tag:
                    self.append(XmlListConfig(element))
            elif element.text:
                text = element.text.strip()
                if text:
                    self.append(text)
            elif element.items():
                self.append(OrderedDict(element.items()))

class XmlDictConfig(dict):
    '''
    Example usage:

    >>> tree = ElementTree.parse('your_file.xml')
    >>> root = tree.getroot()
    >>> xmldict = XmlDictConfig(root)

    Or, if you want to use an XML string:

    >>> root = ElementTree.XML(xml_string)
    >>> xmldict = XmlDictConfig(root)

    And then use xmldict for what it is... a dict.
    '''
    def __init__(self, parent_element):
        childrenNames = []
        for child in parent_element.getchildren():
            childrenNames.append(child.tag)
        
        if parent_element.items():
            self.update(dict(parent_element.items()))
        for element in parent_element:
            if len(element):  # was: if element:
                # treat like dict - we assume that if the first two tags
                # in a series are different, then they are all different.
                if len(element) == 1 or element[0].tag != element[1].tag:
                    aDict = XmlDictConfig(element)
                # treat like list - we assume that if the first two tags
                # in a series are the same, then the rest are the same.
                else:
                    # here, we put the list in dictionary; the key is the
                    # tag name the list elements all share in common, and
                    # the value is the list itself 
                    aDict = {element[0].tag: XmlListConfig(element)}
                # if the tag has attributes, add those to the dict
                if element.items():
                    aDict.update(dict(element.items()))
                    
                if childrenNames.count(element.tag) > 1:
                    try:
                        currentValue = self[element.tag]
                        currentValue.append(aDict)
                        self.update({element.tag: currentValue})
                    except: #the first of its kind, an empty list must be created
                        self.update({element.tag: [aDict]}) #aDict is written in [], i.e. it will be a list

                else:
                    self.update({element.tag: aDict})
            # this assumes that if you've got an attribute in a tag,
            # you won't be having any text. This may or may not be a 
            # good idea -- time will tell. It works for the way we are
            # currently doing XML configuration files...
            elif element.items():
                self.update({element.tag: dict(element.items())})
            # finally, if there are no child tags and no attributes, extract
            # the text
            else:
                self.update({element.tag: element.text})

if len(sys.argv) not in (2,3):
    print("Usage:")
    print("%s xml_in_file" % sys.argv[0])
    print("%s xml_in_file xml_out_file" % sys.argv[0])
    sys.exit(1)

# Convert the XML to a dictionary
tree = ElementTree.parse(sys.argv[1])
root = tree.getroot()
xmldict = XmlDictConfig(root)

# Convenience pointers and sanity checks
image = xmldict['AutoProcScalingContainer']['AutoProcIntegrationContainer']['Image']
integration = xmldict['AutoProcScalingContainer']['AutoProcIntegrationContainer']['AutoProcIntegration']
proc = xmldict['AutoProc']
program = xmldict['AutoProcProgramContainer']['AutoProcProgram']
attachments = xmldict['AutoProcProgramContainer']['AutoProcProgramAttachment']
if isinstance(attachments, dict): # Make it a list regardless
    attachments = [attachments]
scaling = xmldict['AutoProcScalingContainer']['AutoProcScaling']

if proc == None:
    sys.exit("ERROR - please make sure the XML file contains an AutoProc element.")
if scaling == None:
    sys.exit("ERROR - please make sure the XML file contains an AutoProcScaling element.")
if integration == None:
    sys.exit("ERROR - please make sure the XML file contains an AutoProcIntegration element.")
if image == None or image['fileName'] == None or image['fileLocation'] == None:
    sys.exit("ERROR - please make sure the XML file contains an Image element with fileName and fileLocation elements.")

s = [None, None, None]
for i in xrange(0,3):
    stats = xmldict['AutoProcScalingContainer']['AutoProcScalingStatistics'][i]
    if stats['scalingStatisticsType'] == 'outerShell':
        s[0] = stats
    elif stats['scalingStatisticsType'] == 'innerShell':
        s[1] = stats
    elif stats['scalingStatisticsType'] == 'overall':
        s[2] = stats

if s[0] == None or s[1] == None or s[2] == None:
    sys.exit("ERROR - please make sure the XML file contains 3 AutoProcScalingStatistics elements.")

# Get a database cursor
cursor = dbconnection.connect_to_prod()

# Find the datacollection associated with this data reduction run

xml_dir = os.path.split(sys.argv[1])[0]

try:
  dc_id = int(open(os.path.join(xml_dir, '.dc_id'), 'r').read())
  print 'Got DC ID %d from file system' % dc_id
except:
  dc_id = core.retrieve_datacollection_id(cursor, image['fileName'], image['fileLocation'])

# Store results from XIA2 / MX data reduction pipelines
# ...first the program info 
params = mxdatareduction.get_program_params()
if 'processingPrograms' in program:
{'fraction_non_alphanumeric': 0.10256615878107458, 'fraction_numerical': 0.007538091419406576, 'mean_word_length': 2.4364838798567097, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 38}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3592260', 'n_tokens_mistral': 3540, 'n_tokens_neox': 3374, 'n_words': 1097}