<commit_before>from datetime import datetime, timedelta

from redis import Redis
import simplejson

from pylons import app_globals as ag

from rdflib import ConjunctiveGraph
from rdflib import StringInputSource
from rdflib import Namespace, RDF, RDFS, URIRef

from uuid import uuid4

import re

ID_PATTERN = re.compile(r"^[0-9A-z\-\:]+$")

def authz(granary_list,ident):
    g = ag.granary
    g.state.revert()
    g._register_silos()
    granary_list = g.silos
    def _parse_owners(silo_name):
        kw = g.describe_silo(silo_name)
        if "owners" in kw.keys():
            owners = [x.strip() for x in kw['owners'].split(",") if x]
            return owners
        else:
            return []
    
    if ident['role'] == "admin":
        return granary_list
    else:
        authd = []
        for item in granary_list:
            owners = _parse_owners(item)
            if ident['repoze.who.userid'] in owners:
                authd.append(item)
        return authd

def allowable_id(identifier):
    if ID_PATTERN.match(identifier):
        return identifier

def allowable_id2(strg):
    if len(strg) < 2 or ' ' in strg:
        return False
    search=re.compile(r'%s'%ag.naming_rule).search
    return not bool(search(strg))

def is_embargoed(silo, id, refresh=False):
    # TODO evaluate r.expire settings for these keys - popularity resets ttl or increases it?
    r = Redis()
    e = r.get("%s:%s:embargoed" % (silo.state['storage_dir'], id))
    e_d = r.get("%s:%s:embargoed_until" % (silo.state['storage_dir'], id))
    if refresh or (not e or not e_d):
        if silo.exists(id):
            item = silo.get_item(id)
            e = item.metadata.get("embargoed")
            e_d = item.metadata.get("embargoed_until")
            if e not in ['false', 0, False]:
                e = True
            else:
                e = False
            r.set("%s:%s:embargoed" % (silo.state['storage_dir'], id), e)
            r.set("%s:%s:embargoed_until" % (silo.state['storage_dir'], id), e_d)
    return (e, e_d)

def create_new(silo, id, creator, title=None, embargoed=True, embargoed_until=None, embargo_days_from_now=None, **kw):
    item = silo.get_item(id)
    item.metadata['createdby'] = creator
    item.metadata['embargoed'] = embargoed
    item.metadata['uuid'] = uuid4().hex
    item.add_namespace('oxds', "http://vocab.ox.ac.uk/dataset/schema#")
    item.add_triple(item.uri, u"rdf:type", "oxds:DataSet")

    if embargoed:
        if embargoed_until:
            embargoed_until_date = embargoed_until
        elif embargo_days_from_now:
            embargoed_until_date = (datetime.now() + timedelta(days=embargo_days_from_now)).isoformat()
        else:
            embargoed_until_date = (datetime.now() + timedelta(days=365*70)).isoformat()
        item.metadata['embargoed_until'] = embargoed_until_date
        item.add_triple(item.uri, u"oxds:isEmbargoed", 'True')
        item.add_triple(item.uri, u"oxds:embargoedUntil", embargoed_until_date)
    else:
        item.add_triple(item.uri, u"oxds:isEmbargoed", 'False')
    item.add_triple(item.uri, u"dcterms:identifier", id)
    item.add_triple(item.uri, u"dcterms:creator", creator)   
    item.add_triple(item.uri, u"dcterms:created", datetime.now())
    item.add_triple(item.uri, u"oxds:currentVersion", item.currentversion)
    
    #TODO: Add current version metadata
    if title:
        item.add_triple(item.uri, u"rdfs:label", title)
    item.sync()
    return item

def get_readme_text(item, filename="README"):
    with item.get_stream(filename) as fn:
        text = fn.read().decode("utf-8")
    return u"%s\n\n%s" % (filename, text)

def test_rdf(text, isstring=True):
    g = ConjunctiveGraph()
    if isstring:
        text = StringInputSource(text)
    try:
        g = g.parse(text, format='xml')
        return True
    except:
        return False

def munge_manifest(manifest_str, item, manifest_type='http://vocab.ox.ac.uk/dataset/schema#Grouping', isstring=True):    
    #Get triples from the manifest file and remove the file
    triples = None
    ns = None
    ns, triples, seeAlsoFiles = read_manifest(item.uri, manifest_str, manifest_type=manifest_type, isstring=isstring)
    #item.add_namespace('owl', "http://www.w3.org/2002/07/owl#")
    if ns and triples:
        for k, v in ns.iteritems():
            item.add_namespace(k, v)
        for (s, p, o) in triples:
            if str(p) == 'http://purl.org/dc/terms/title':
                item.del_triple(item.uri, u"dcterms:title")    
            item.add_triple(s, p, o)
    item.sync()
    #if seeAlsoFiles:
    #    for filename in seeAlsoFiles:
    #        #get path to filename
    #        filepath = None
    #        if item.isfile(filepath):
    #            with item.get_stream(filepath) as fn:
    #                text = fn.read()
    #            self.munge_manifest(text, item, manifest_type=manifest_type)
    return True

def read_manifest(target_dataset_uri, manifest_str, manifest_type='http://vocab.ox.ac.uk/dataset/schema#Grouping', isstring=True):
    triples = []
    namespaces = {}
    seeAlsoFiles = []
    oxdsClasses = ['http://vocab.ox.ac.uk/dataset/schema#Grouping', 'http://vocab.ox.ac.uk/dataset/schema#DataSet']
    if isstring:
        manifest_str = StringInputSource(manifest_str)
    g = ConjunctiveGraph()
    gparsed = g.parse(manifest_str, format='xml')
    namespaces = dict(g.namespaces())
    datasetType = False
    for s,p,o in gparsed.triples((None, RDF.type, None)):
        if str(o) == manifest_type:
            datasetType = True
            if s.startswith('http'):
                namespaces['owl'] = URIRef("http://www.w3.org/2002/07/owl#")
                triples.append((target_dataset_uri, 'owl:sameAs', s))
        elif str(o) in oxdsClasses and not s.startswith('http'):
            gparsed.remove((s, p, o))
    #for s,p,o in gparsed.triples((None, RDFS.seeAlso, None)):
    #    seeAlsoFiles.append(str(o))
    for s,p,o in gparsed.triples((None, None, None)):
        if str(p) == 'http://www.w3.org/2000/01/rdf-schema#seeAlso' and str(o):
            seeAlsoFiles.append(str(o))
        elif datasetType or not s.startswith('http'):
            triples.append((target_dataset_uri, p, o))<commit_msg>rdf from other files are munged, if added using the rdfs:seeAlso property<commit_after>from datetime import datetime, timedelta

from redis import Redis
import simplejson

from pylons import app_globals as ag

from rdflib import ConjunctiveGraph
from rdflib import StringInputSource
from rdflib import Namespace, RDF, RDFS, URIRef

from uuid import uuid4
import re

ID_PATTERN = re.compile(r"^[0-9A-z\-\:]+$")

def authz(granary_list,ident):
    g = ag.granary
    g.state.revert()
    g._register_silos()
    granary_list = g.silos
    def _parse_owners(silo_name):
        kw = g.describe_silo(silo_name)
        if "owners" in kw.keys():
            owners = [x.strip() for x in kw['owners'].split(",") if x]
            return owners
        else:
            return []
    
    if ident['role'] == "admin":
        return granary_list
    else:
        authd = []
        for item in granary_list:
            owners = _parse_owners(item)
            if ident['repoze.who.userid'] in owners:
                authd.append(item)
        return authd

def allowable_id(identifier):
    if ID_PATTERN.match(identifier):
        return identifier

def allowable_id2(strg):
    if len(strg) < 2 or ' ' in strg:
        return False
    search=re.compile(r'%s'%ag.naming_rule).search
    return not bool(search(strg))

def is_embargoed(silo, id, refresh=False):
    # TODO evaluate r.expire settings for these keys - popularity resets ttl or increases it?
    r = Redis()
    e = r.get("%s:%s:embargoed" % (silo.state['storage_dir'], id))
    e_d = r.get("%s:%s:embargoed_until" % (silo.state['storage_dir'], id))
    if refresh or (not e or not e_d):
        if silo.exists(id):
            item = silo.get_item(id)
            e = item.metadata.get("embargoed")
            e_d = item.metadata.get("embargoed_until")
            if e not in ['false', 0, False]:
                e = True
            else:
                e = False
            r.set("%s:%s:embargoed" % (silo.state['storage_dir'], id), e)
            r.set("%s:%s:embargoed_until" % (silo.state['storage_dir'], id), e_d)
    return (e, e_d)

def create_new(silo, id, creator, title=None, embargoed=True, embargoed_until=None, embargo_days_from_now=None, **kw):
    item = silo.get_item(id)
    item.metadata['createdby'] = creator
    item.metadata['embargoed'] = embargoed
    item.metadata['uuid'] = uuid4().hex
    item.add_namespace('oxds', "http://vocab.ox.ac.uk/dataset/schema#")
    item.add_triple(item.uri, u"rdf:type", "oxds:DataSet")

    if embargoed:
        if embargoed_until:
            embargoed_until_date = embargoed_until
        elif embargo_days_from_now:
            embargoed_until_date = (datetime.now() + timedelta(days=embargo_days_from_now)).isoformat()
        else:
            embargoed_until_date = (datetime.now() + timedelta(days=365*70)).isoformat()
        item.metadata['embargoed_until'] = embargoed_until_date
        item.add_triple(item.uri, u"oxds:isEmbargoed", 'True')
        item.add_triple(item.uri, u"oxds:embargoedUntil", embargoed_until_date)
    else:
        item.add_triple(item.uri, u"oxds:isEmbargoed", 'False')
    item.add_triple(item.uri, u"dcterms:identifier", id)
    item.add_triple(item.uri, u"dcterms:creator", creator)   
    item.add_triple(item.uri, u"dcterms:created", datetime.now())
    item.add_triple(item.uri, u"oxds:currentVersion", item.currentversion)
    
    #TODO: Add current version metadata
    if title:
        item.add_triple(item.uri, u"rdfs:label", title)
    item.sync()
    return item

def get_readme_text(item, filename="README"):
    with item.get_stream(filename) as fn:
        text = fn.read().decode("utf-8")
    return u"%s\n\n%s" % (filename, text)

def test_rdf(text, isstring=True):
    g = ConjunctiveGraph()
    if isstring:
        text = StringInputSource(text)
    try:
        g = g.parse(text, format='xml')
        return True
    except:
        return False

def munge_manifest(manifest_str, item, manifest_type='http://vocab.ox.ac.uk/dataset/schema#Grouping', isstring=True):    
    #Get triples from the manifest file and remove the file
    triples = None
    ns = None
    seeAlsoFiles = None
    ns, triples, seeAlsoFiles = read_manifest(item.uri, manifest_str, manifest_type=manifest_type, isstring=isstring)
    if ns and triples:
        for k, v in ns.iteritems():
            item.add_namespace(k, v)
        for (s, p, o) in triples:
            if str(p) == 'http://purl.org/dc/terms/title':
                item.del_triple(item.uri, u"dcterms:title")    
            item.add_triple(s, p, o)
    item.sync()
    if seeAlsoFiles:
        tmp_m = item.get_rdf_manifest()
        for filename in seeAlsoFiles:
            fullfilepath = None
            #get path to filename
            for (s, p, o) in tmp_m.get_graph():
                if str(p) == 'http://www.openarchives.org/ore/terms/aggregates' and filename in str(o):
                    filepath = str(o).replace(item.uri, '').strip().lstrip('/')
                    fullfilepath = item.to_dirpath(filepath=filepath)
                    break
            if fullfilepath and item.isfile(fullfilepath):
                with item.get_stream(filepath) as fn:
                    text = fn.read()
                if test_rdf(text):
                    munge_manifest(text, item, manifest_type=manifest_type)
    return True

def read_manifest(target_dataset_uri, manifest_str, manifest_type='http://vocab.ox.ac.uk/dataset/schema#Grouping', isstring=True):
    triples = []
    namespaces = {}
    seeAlsoFiles = []
    oxdsClasses = ['http://vocab.ox.ac.uk/dataset/schema#Grouping', 'http://vocab.ox.ac.uk/dataset/schema#DataSet']
    if isstring:
        manifest_str = StringInputSource(manifest_str)
    g = ConjunctiveGraph()
    gparsed = g.parse(manifest_str, format='xml')
    namespaces = dict(g.namespaces())
    datasetType = False
    for s,p,o in gparsed.triples((None, RDF.type, None)):
        if str(o) == manifest_type:
            datasetType = True
            if s.startswith('http'):
                namespaces['owl'] = URIRef("http://www.w3.org/2002/07/owl#")
                triples.append((target_dataset_uri, 'owl:sameAs', s))
        elif str(o) in oxdsClasses and not s.startswith('http'):
            gparsed.remove((s, p, o))
    for s,p,o in gparsed.triples((None, None, None)):
        if str(p) == 'http://www.w3.org/2000/01/rdf-schema#seeAlso' and str(o):
            seeAlsoFiles.append(str(o))
        elif datasetType or not s.startswith('http'):
            triples.append((target_dataset_uri, p, o))
{'fraction_non_alphanumeric': 0.13150300288589034, 'fraction_numerical': 0.004757819202870291, 'mean_word_length': 2.704709621496677, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2193603', 'n_tokens_mistral': 4451, 'n_tokens_neox': 4349, 'n_words': 902}