<commit_before>'''
Created on Aug 13, 2013

@author: adam
'''

import re
import json

def ExtractUnitsFromMSL(p_mslSIUnits):
    # Read contents of SIunits.mo
    f_mslSIUnits = open(p_mslSIUnits,'r')
    as_mslSIUnits_ = f_mslSIUnits.readlines()
    f_mslSIUnits.close()
        
    # Remove comment lines
    s_mslSIUnits = ''
    for s in as_mslSIUnits_:
        if s.find('//') == -1:
            s_mslSIUnits += s
        
    # Strip out whitespace
    s_mslSIUnits = s_mslSIUnits.replace('\n','').replace('\r','').replace('\t','').replace('  ','')
    
    # Divide into lines based on semicolon
    d_TypeStatements = dict()
    as_Statements = s_mslSIUnits.split(';')
    pr_IsTypeStatement = re.compile('type.*')
    for s in as_Statements:
        
        mo_IsTypeStatement = pr_IsTypeStatement.match(s)
        if mo_IsTypeStatement != None:
            mo = re.match(r"type (?P<type_name>\w+)",s)
            key = mo.group(1)
            
            d_TypeStatements[key] = GetUnitPropDict(s)
            d_TypeStatements[key]['ClassPath'] = 'Modelica.SIunits.' + key
                    
    return d_TypeStatements
            
def CombineDict(d1, d2):
    return dict(d1.items() + d2.items())

# RegEXes for unit properties
apr_DataTypeProps = []
apr_DataTypeProps.append( re.compile('.*=\s*(?P<UnitDataType>\w+) ') )
apr_DataTypeProps.append( re.compile('.*final quantity\s*=\s*\"(?P<UnitQuantity>\w+)\"') )
apr_DataTypeProps.append( re.compile('.*final unit\s*=\s*\"(?P<UnitUnit>\w+)\"') )
apr_DataTypeProps.append( re.compile('.*displayUnit\s*=\s*\"(?P<UnitDisplayUnit>\w+)\"') )
apr_DataTypeProps.append( re.compile('.*min\s*=\s*(?P<UnitMin>\w+)') )
apr_DataTypeProps.append( re.compile('.*start\s*=\s*(?P<UnitStart>\w+)') )
pr_EquivalentClass = re.compile('type \S*\s*=\s*(?P<EquivalentClass>\w+)')
def GetUnitPropDict(str):
    d_rtn = dict()
        
    for pr in apr_DataTypeProps:
        if pr.match(str) != None:
            d_rtn = CombineDict(d_rtn,pr.match(str).groupdict())
    
    if str.find('(') == -1:
        # This is just equivalent to something else
        d_rtn = CombineDict(d_rtn, pr_EquivalentClass.match(str).groupdict())
    
    return d_rtn

def GenerateOWLNode(s_Name, d_Unit):
    xml_OWLNode = '\n  <NamedIndividual rdf:about="http://metamorphinc.com/ontology/modelica/individuals#' + s_Name + '">\n'
    xml_OWLNode += '    <rdf:type rdf:resource="&modelica;ModelicaUnitClass"/>\n'
    for key,value in d_Unit.iteritems():
        xml_OWLNode += '    <modelica:' + key + ' rdf:datatype="&xsd;string">' + value + "</modelica:" + key + ">\n"
    xml_OWLNode += '  </NamedIndividual>\n'
    
    return xml_OWLNode

def GenerateOWLIndividualFileContent(d_Units):
    xml_Header = '''<?xml version="1.0"?>

<!DOCTYPE rdf:RDF [
    <!ENTITY owl "http://www.w3.org/2002/07/owl#" >
    <!ENTITY xsd "http://www.w3.org/2001/XMLSchema#" >
    <!ENTITY rdfs "http://www.w3.org/2000/01/rdf-schema#" >
    <!ENTITY modelica "http://metamorphinc.com/ontology/modelica#" >
    <!ENTITY rdf "http://www.w3.org/1999/02/22-rdf-syntax-ns#" >
]>
    <rdf:RDF xmlns="http://www.w3.org/2002/07/owl#"
     xml:base="http://www.w3.org/2002/07/owl"
     xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#"
     xmlns:modelica="http://metamorphinc.com/ontology/modelica#"
     xmlns:owl="http://www.w3.org/2002/07/owl#"
     xmlns:xsd="http://www.w3.org/2001/XMLSchema#"
     xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
    <Ontology rdf:about="http://metamorphinc.com/ontology/modelica/individuals">
        <imports rdf:resource="http://metamorphinc.com/ontology/modelica"/>
    </Ontology>'''
    
    xml_Footer = '''
</rdf:RDF>'''
    
    xml_Final = xml_Header
    for key, value in d_Units.iteritems():
        xml_Final += GenerateOWLNode(key,value)
    xml_Final += xml_Footer
    
    return xml_Final

def GenerateOWLIndividualFile(p_SIUnits,p_OutputFile):
    d_TypeStatements = ExtractUnitsFromMSL(p_SIUnits)
    
    xml_IndividualFile = GenerateOWLIndividualFileContent(d_TypeStatements)
    f = open(p_OutputFile,'w')
    f.write(xml_IndividualFile)
    f.close()<commit_msg>replacing re parser with json ast file<commit_after>#!/usr/bin/env python
import re
import json

def ExtractUnitsFromJson(jsonSIUnitsFile):
    ''' Read contents of SIunits json file '''
    f = open(jsonSIUnitsFile,'r')
    jsonSIUnits = json.load(f)
    
    mapping = {'quantity':'UnitQuantity', 'unit':'UnitUnit',
               'displayUnit':'UnitDisplayUnit', 'min':'UnitMin', 
               'start':'UnitStart'}
    for unit in jsonSIUnits:
       # print unit
        result = {}
        result['ClassPath'] = unit['name']
        if result['ClassPath'] == 'Modelica.SIunits': continue
        unitName = re.match('Modelica.SIunits.(.+)',result['ClassPath']).group(1)
        if unit['modifiers']:
            result['UnitDataType'] = unit['modifiers'][0]['className']
            for element in unit['modifiers'][0]['values']:
                m = mapping.get(element['name'],None)
                if not m == None:
                    result[m] = element['value']
        #print unitName,result
        yield (unitName,result)
    
    f.close()
        
     

def GenerateOWLNode(s_Name, d_Unit):
    xml_OWLNode = '\n  <NamedIndividual rdf:about="http://metamorphinc.com/ontology/modelica/individuals#' + s_Name + '">\n'
    xml_OWLNode += '    <rdf:type rdf:resource="&modelica;ModelicaUnitClass"/>\n'
    for key,value in d_Unit.iteritems():
        xml_OWLNode += '    <modelica:' + key + ' rdf:datatype="&xsd;string">' + value + "</modelica:" + key + ">\n"
    xml_OWLNode += '  </NamedIndividual>\n'
    
    return xml_OWLNode

def GenerateOWLIndividualFileContent(d_Units):
    xml_Header = '''<?xml version="1.0"?>

<!DOCTYPE rdf:RDF [
    <!ENTITY owl "http://www.w3.org/2002/07/owl#" >
    <!ENTITY xsd "http://www.w3.org/2001/XMLSchema#" >
    <!ENTITY rdfs "http://www.w3.org/2000/01/rdf-schema#" >
    <!ENTITY modelica "http://metamorphinc.com/ontology/modelica#" >
    <!ENTITY rdf "http://www.w3.org/1999/02/22-rdf-syntax-ns#" >
]>
    <rdf:RDF xmlns="http://www.w3.org/2002/07/owl#"
     xml:base="http://www.w3.org/2002/07/owl"
     xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#"
     xmlns:modelica="http://metamorphinc.com/ontology/modelica#"
     xmlns:owl="http://www.w3.org/2002/07/owl#"
     xmlns:xsd="http://www.w3.org/2001/XMLSchema#"
     xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
    <Ontology rdf:about="http://metamorphinc.com/ontology/modelica/individuals">
        <imports rdf:resource="http://metamorphinc.com/ontology/modelica"/>
    </Ontology>'''
    
    xml_Footer = '''
</rdf:RDF>'''
    
    xml_Final = xml_Header
    for key, value in d_Units:
        xml_Final += GenerateOWLNode(key,value)
    xml_Final += xml_Footer
    
    return xml_Final

def GenerateOWLIndividualFile(jsonSIUnitsFile,OutputFile):
    d_TypeStatements = ExtractUnitsFromJson(jsonSIUnitsFile)
    print d_TypeStatements
    xml_IndividualFile = GenerateOWLIndividualFileContent(d_TypeStatements)
    with open(OutputFile,'w') as f:
        f.write(xml_IndividualFile)
        
GenerateOWLIndividualFile('modelica_units.json','example.xml')

{'fraction_non_alphanumeric': 0.1701715550636414, 'fraction_numerical': 0.02213613724405091, 'mean_word_length': 3.183449074074074, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 2, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 20, 'xml': 47}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '309074', 'n_tokens_mistral': 2816, 'n_tokens_neox': 2667, 'n_words': 375}