<commit_before>'''
Created on 16 Sep 2014

@author: s05nc4
'''

from dot.rural.sepake.csv_to_rdf import CSV
from rdflib.namespace import FOAF, XSD
from dot.rural.sepake.ontology import SEPAKE, SEPAKEOntologyGraph, PROV
from rdflib import RDF, RDFS
from rdflib import Graph
from csv_to_rdf import CSVGraph

class UKEOFGraph(Graph):
    def __init__(self, include_ontology = True):
        super(UKEOFGraph, self).__init__()
        if include_ontology:
            self += SEPAKEOntologyGraph()
        csv = CSVGraph(include_ontology)
        print 'Downloading data from UKEOF...'
        csv.read_url('https://catalogue.ukeof.org.uk/api/documents?format=csv')
        self += csv
        for sparql in [INSERT_TYPE(), 
                       INSERT_LABEL(), 
                       INSERT_HOMEPAGE(), 
                       INSERT_LEAD_ORG(), 
                       INSERT_START_DATE(), 
                       INSERT_END_DATE(),
                       INSERT_COMMENT()]:
            print 'Updating with %s...' % sparql
            self.update(sparql)
        
def _expand(template_func):
    def expanded():
        template = template_func()
        return template.format(csv = CSV,
                               xsd = XSD,
                               rdf = RDF, 
                               rdfs = RDFS, 
                               prov = PROV, 
                               foaf = FOAF,
                               sepake = SEPAKE)
    return expanded

@_expand
def INSERT_TYPE():
    return '''
INSERT {{
    ?urilink <{rdf.type}> <{sepake.UKEOFActivity}> .
    ?urilink <{prov.wasDerivedFrom}> ?row .
}}
WHERE {{
    ?row <{rdf.type}> <{csv.Row}> .
    ?row <{rdfs.member}> ?typecell .<commit_msg>Added some more verbosity For #9-semweb-model<commit_after>'''
Created on 16 Sep 2014

@author: s05nc4
'''

from dot.rural.sepake.csv_to_rdf import CSV
from rdflib.namespace import FOAF, XSD
from dot.rural.sepake.ontology import SEPAKE, SEPAKEOntologyGraph, PROV
from rdflib import RDF, RDFS
from rdflib import Graph
from csv_to_rdf import CSVGraph

class UKEOFGraph(Graph):
    def __init__(self, include_ontology = True):
        super(UKEOFGraph, self).__init__()
        if include_ontology:
            self += SEPAKEOntologyGraph()
        csv = CSVGraph(include_ontology)
        self._log_len()
        print 'Downloading data from UKEOF...'
        csv.read_url('https://catalogue.ukeof.org.uk/api/documents?format=csv')
        self += csv
        self._log_len()
        for sparql in [INSERT_TYPE(), 
                       INSERT_LABEL(), 
                       INSERT_HOMEPAGE(), 
                       INSERT_LEAD_ORG(), 
                       INSERT_START_DATE(), 
                       INSERT_END_DATE(),
                       INSERT_COMMENT()]:
            print 'Updating with %s...' % sparql
            self.update(sparql)
            self._log_len()
        
    def _log_len(self):
        print 'Accumulated %d triples' % len(self)
        
def _expand(template_func):
    def expanded():
        template = template_func()
        return template.format(csv = CSV,
                               xsd = XSD,
                               rdf = RDF, 
                               rdfs = RDFS, 
                               prov = PROV, 
                               foaf = FOAF,
                               sepake = SEPAKE)
    return expanded

@_expand
def INSERT_TYPE():
    return '''
INSERT {{
    ?urilink <{rdf.type}> <{sepake.UKEOFActivity}> .
    ?urilink <{prov.wasDerivedFrom}> ?row .
}}
WHERE {{
    ?row <{rdf.type}> <{csv.Row}> .
    ?row <{rdfs.member}> ?typecell .
{'fraction_non_alphanumeric': 0.10296540362438221, 'fraction_numerical': 0.005216913783635365, 'mean_word_length': 1.6379435191889935, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1202555', 'n_tokens_mistral': 1184, 'n_tokens_neox': 1077, 'n_words': 245}