<commit_before>"""
    cvScript:
        Downloads and loads Controlled Vocabulary terms from the Vocabulary's online api and populates a empty ODM2 database

"""

__author__ = 'jmeline'


import cvmodels
import xml.etree.ElementTree as ET
import pprint
pp = pprint.PrettyPrinter(indent=8)

from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
try:
    # python 3
    import urllib.request as request
    basestring = str

except:
    # python 2
    import urllib as request

####################################
#       Custom object to hold the values
####################################
class CVObject:
    def __init__(self, term=None, name=None, defi=None, cat=None, uri=None):
        self.Term = term
        self.Name = name
        self.Definition = defi
        self.Category = cat
        self.SourceVocabularyURI = uri

    def __repr__ (self):
        return "<cvObject('%s', '%s', '%s', '%s', '%s'>" % (
            self.Term, self.Name, self.Definition,
            self.Category, self.SourceVocabularyURI)

    def get_elements(self):
        return {"Term": self.Term,
                "Name": self.Name,
                "Definition": self.Definition,
                "Category": self.Category,
                "SourceVocabularyURI": self.SourceVocabularyURI
            }

########################################
#       List of CV's:
#           Note: This must be updated if more types are introduced
########################################

vocab= ["actiontype", "qualitycode", "samplingfeaturegeotype", "elevationdatum",
        "resulttype", "sampledmedium", "speciation", "aggregationstatistic",
        "methodtype", "taxonomicclassifiertype", "sitetype", "censorcode",
        "directivetype", "datasettype", "organizationtype", "status",
        "annotationtype", "samplingfeaturetype", "equipmenttype", "specimenmedium",
        "spatialoffsettype", "referencematerialmedium", "specimentype",
        "variabletype", "variablename", "actiontype", "propertydatatype"]

objects = ['CVActionType', 'CVQualityCode', 'CVSamplingFeatureGeoType', 'CVElevationDatum','CVResultType', 'CVSampledMedium',
    'CVSpeciation', 'CVAggregationStatistic', 'CVMethodType', 'CVTaxonomicClassifierType', 'CVSiteType', 'CVCensorCode',
    'CVDirectiveType', 'CVDatasetType', 'CVOrganizationType', 'CVStatus', 'CVAnnotationType', 'CVSamplingFeatureType',
    'CVEquipmentType', 'CVSpecimenMedium','CVSpatialOffsetType', 'CVReferenceMaterialMedium', 'CVSpecimenType',
    'CVVariableType', 'CVVariableName', 'CVActionType', 'CVActionType', 'CVPropertyDataType']
####################################

dictionary = list(zip(vocab, objects))

####################################
#       Urls and skos links
####################################

url ="http://vocabulary.odm2.org/api/v1/%s/?format=skos"
dc="{http://purl.org/dc/elements/1.1/}%s"
rdf="{http://www.w3.org/1999/02/22-rdf-syntax-ns#}%s"
skos="{http://www.w3.org/2004/02/skos/core#}%s"
odm2="{http://vocabulary.odm2.org/ODM2/ODM2Terms/}%s"

conn_string = 'mysql+pymysql://root:zxc@localhost/ODM2'
engine = create_engine(conn_string,echo=False, encoding='utf-8')
Session = sessionmaker(bind=engine)
session = Session()

####################################
#       Load up all the cvmodels into a list
####################################
modules = []
def load_up_cvmodels():
    model = [cvmodels]
    assert model
    for item in model:
        if isinstance(item, basestring):
            modules.append(__import__(item))
        else:
            modules.append(item)

####################################
#       Converts string class names into functions
####################################
def get_klass(klass_name):
    klass = None
    for module in modules:
        klass = getattr(module, klass_name)
        break
    return klass

####################################
#       Obtain xml_data from vocabulary website
####################################
def obtain_url(vocab):
    vocab_site = url % vocab
    try:
        xml_data = request.urlopen(vocab_site).read()
        return xml_data
    except HTTPError as e:
        print ("There was an error with connecting to the site: ", e)

#####################################
#       Converts a cv_object to the respective SQLAlchemy object
#####################################
def build_sql_object(cv_object, str_obj):
        _class_object = get_klass(str_obj)
        return _class_object(**cv_object.get_elements())

#####################################
#       Parses xml_data and creates cv_obj
#####################################
def build_cv_object(item):
        cv_obj = CVObject()

        try:
            cv_obj.Name = item.find(skos % "prefLabel").text
        except:
            return None
        cv_obj.Term = item.attrib[rdf % "about"].split('/')[-1]
        cv_obj.Definition = item.find(skos % "definition").text
        cv_obj.SourceVocabularyURI = item.attrib[rdf % 'about']
        try:
            cv_obj.Category = item.find(odm2 % 'category').text
        except:
            pass

        return cv_obj

#####################################
#       Loads sql_objects into SQLAlchemy session
#####################################
def load_up_objects():
    for vocab, obj in dictionary:
        objs = []
        xml_data = obtain_url(vocab)
        root = ET.fromstring(xml_data)
        for item in root.findall(rdf % 'Description'):
            cv_obj = build_cv_object(item)
            if not cv_obj:
                continue
            # print("TableName: ", sql_obj.__tablename__)
            sql_obj = build_sql_object(cv_obj, obj)
            session.add(sql_obj)
            objs.append(sql_obj)

        try:
            output = session.new
            session.commit()
            print ("Session: ", output)
        except Exception as e:
            session.rollback()
            pass

if __name__ == '__main__':
    load_up_cvmodels()
    assert len(modules) > 0

    load_up_objects()<commit_msg>Adding a progressbar to cvscript<commit_after>"""
    cvScript:
        Downloads and loads Controlled Vocabulary terms from the Vocabulary's online api and populates a empty ODM2 database

"""

__author__ = 'jmeline'


import cvmodels
import xml.etree.ElementTree as ET
import pprint
pp = pprint.PrettyPrinter(indent=8)

from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from progressbar import ProgressBar, Percentage, Bar, RotatingMarker
try:
    # python 3
    import urllib.request as request
    basestring = str

except:
    # python 2
    import urllib as request


conn_string = 'mysql+pymysql://root:zxc@localhost/ODM2'
####################################
#       Custom object to hold the values
####################################
class CVObject:
    def __init__(self, term=None, name=None, defi=None, cat=None, uri=None):
        self.Term = term
        self.Name = name
        self.Definition = defi
        self.Category = cat
        self.SourceVocabularyURI = uri

    def __repr__ (self):
        return "<cvObject('%s', '%s', '%s', '%s', '%s'>" % (
            self.Term, self.Name, self.Definition,
            self.Category, self.SourceVocabularyURI)

    def get_elements(self):
        return {"Term": self.Term,
                "Name": self.Name,
                "Definition": self.Definition,
                "Category": self.Category,
                "SourceVocabularyURI": self.SourceVocabularyURI
            }

########################################
#       List of CV's:
#           Note: This must be updated if more types are introduced
########################################

vocab= ["actiontype", "qualitycode", "samplingfeaturegeotype", "elevationdatum",
        "resulttype", "sampledmedium", "speciation", "aggregationstatistic",
        "methodtype", "taxonomicclassifiertype", "sitetype", "censorcode",
        "directivetype", "datasettype", "organizationtype", "status",
        "annotationtype", "samplingfeaturetype", "equipmenttype", "specimenmedium",
        "spatialoffsettype", "referencematerialmedium", "specimentype",
        "variabletype", "variablename", "actiontype", "propertydatatype"]

objects = ['CVActionType', 'CVQualityCode', 'CVSamplingFeatureGeoType', 'CVElevationDatum','CVResultType', 'CVSampledMedium',
    'CVSpeciation', 'CVAggregationStatistic', 'CVMethodType', 'CVTaxonomicClassifierType', 'CVSiteType', 'CVCensorCode',
    'CVDirectiveType', 'CVDatasetType', 'CVOrganizationType', 'CVStatus', 'CVAnnotationType', 'CVSamplingFeatureType',
    'CVEquipmentType', 'CVSpecimenMedium','CVSpatialOffsetType', 'CVReferenceMaterialMedium', 'CVSpecimenType',
    'CVVariableType', 'CVVariableName', 'CVActionType', 'CVActionType', 'CVPropertyDataType']
####################################

dictionary = list(zip(vocab, objects))

####################################
#       Urls and skos links
####################################

url ="http://vocabulary.odm2.org/api/v1/%s/?format=skos"
dc="{http://purl.org/dc/elements/1.1/}%s"
rdf="{http://www.w3.org/1999/02/22-rdf-syntax-ns#}%s"
skos="{http://www.w3.org/2004/02/skos/core#}%s"
odm2="{http://vocabulary.odm2.org/ODM2/ODM2Terms/}%s"

engine = create_engine(conn_string,echo=False, encoding='utf-8')
Session = sessionmaker(bind=engine)
session = Session()

####################################
#       Load up all the cvmodels into a list
####################################
modules = []
def load_up_cvmodels():
    model = [cvmodels]
    assert model
    for item in model:
        if isinstance(item, basestring):
            modules.append(__import__(item))
        else:
            modules.append(item)

####################################
#       Converts string class names into functions
####################################
def get_klass(klass_name):
    klass = None
    for module in modules:
        klass = getattr(module, klass_name)
        break
    return klass

####################################
#       Obtain xml_data from vocabulary website
####################################
def obtain_url(vocab):
    vocab_site = url % vocab
    try:
        xml_data = request.urlopen(vocab_site).read()
        return xml_data
    except Exception as e:
        print ("There was an error with connecting to the site: ", e)

#####################################
#       Converts a cv_object to the respective SQLAlchemy object
#####################################
def build_sql_object(cv_object, str_obj):
        _class_object = get_klass(str_obj)
        return _class_object(**cv_object.get_elements())

#####################################
#       Parses xml_data and creates cv_obj
#####################################
def build_cv_object(item):
        cv_obj = CVObject()

        try:
            cv_obj.Name = item.find(skos % "prefLabel").text
        except:
            return None
        cv_obj.Term = item.attrib[rdf % "about"].split('/')[-1]
        cv_obj.Definition = item.find(skos % "definition").text
        cv_obj.SourceVocabularyURI = item.attrib[rdf % 'about']
        try:
            cv_obj.Category = item.find(odm2 % 'category').text
        except:
            pass

        return cv_obj

#####################################
#       Loads sql_objects into SQLAlchemy session
#####################################
import sys
def load_up_objects():
    pbar = ProgressBar(widgets=[Percentage(), Bar()], maxval=len(dictionary)).start()
    for i, (vocab, obj) in enumerate(dictionary):
        objs = []
        xml_data = obtain_url(vocab)
        root = ET.fromstring(xml_data)
        for item in root.findall(rdf % 'Description'):
            cv_obj = build_cv_object(item)
            if not cv_obj:
                continue
            # print("TableName: ", sql_obj.__tablename__)
            sql_obj = build_sql_object(cv_obj, obj)
            session.add(sql_obj)
            objs.append(sql_obj)

        try:
            output = session.new
            # session.commit()
            #pp.pprint(output)
        except Exception as e:
            session.rollback()
            pass
        sys.stdout.write('%s\n' % vocab)
        sys.stdout.flush()
        pbar.update(i)
    pbar.finish()

if __name__ == '__main__':
    load_up_cvmodels()
    assert len(modules) > 0

    load_up_objects()
{'fraction_non_alphanumeric': 0.2165718927701056, 'fraction_numerical': 0.0053614947197400484, 'mean_word_length': 3.384259259259259, 'pattern_counts': {'":': 10, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 14}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '799586', 'n_tokens_mistral': 3640, 'n_tokens_neox': 3479, 'n_words': 772}