<commit_before>"""
#-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=#
  This file is part of the Smart Developer Hub Project:
    http://www.smartdeveloperhub.org

  Center for Open Middleware
        http://www.centeropenmiddleware.com/
#-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=#
  Copyright (C) 2015 Center for Open Middleware.
#-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=#
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at 

            http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
#-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=#
"""

from rdflib import ConjunctiveGraph, URIRef, Literal, XSD
from sdh.curator.server import app
import logging
import calendar
from datetime import datetime as dt
from sdh.curator.store import r
import os
import shutil
import shortuuid
from concurrent.futures import ThreadPoolExecutor

__author__ = '<NAME>'

log = logging.getLogger('sdh.curator.store.triples')
pool = ThreadPoolExecutor(max_workers=4)


def load_stream_triples(fid, until):
    def __triplify(x):
        def __term(elm):
            if elm.startswith('<'):
                return URIRef(elm.lstrip('<').rstrip('>'))
            elif '^^' in elm:
                (value, ty) = tuple(elm.split('^^'))
                return Literal(value.replace('"', ''), datatype=URIRef(ty.lstrip('<').rstrip('>')))
            else:
                return Literal(elm.replace('"', ''), datatype=XSD.string)

        c, s, p, o = eval(x)
        return c, __term(s), __term(p), __term(o)

    for x in r.zrangebyscore('fragments:{}:stream'.format(fid), '-inf', '{}'.format(float(until))):
        yield __triplify(x)


def add_stream_triple(fid, tp, (s, p, o), timestamp=None):
    if timestamp is None:
        timestamp = calendar.timegm(dt.utcnow().timetuple())
    quad = (tp, s.n3(), p.n3(), o.n3())
    stream_key = 'fragments:{}:stream'.format(fid)
    not_found = not bool(r.zscore(stream_key, quad))
    if not_found:
        with r.pipeline() as pipe:
            pipe.zadd(stream_key, timestamp, quad)
            pipe.execute()<commit_msg>Use BNodes when a node starts with _:<commit_after>"""
#-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=#
  This file is part of the Smart Developer Hub Project:
    http://www.smartdeveloperhub.org

  Center for Open Middleware
        http://www.centeropenmiddleware.com/
#-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=#
  Copyright (C) 2015 Center for Open Middleware.
#-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=#
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at 

            http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
#-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=#
"""

from rdflib import ConjunctiveGraph, URIRef, Literal, XSD, BNode
from sdh.curator.server import app
import logging
import calendar
from datetime import datetime as dt
from sdh.curator.store import r
import os
import shutil
import shortuuid
from concurrent.futures import ThreadPoolExecutor

__author__ = '<NAME>'

log = logging.getLogger('sdh.curator.store.triples')
pool = ThreadPoolExecutor(max_workers=4)


def load_stream_triples(fid, until):
    def __triplify(x):
        def __term(elm):
            if elm.startswith('<'):
                return URIRef(elm.lstrip('<').rstrip('>'))
            elif '^^' in elm:
                (value, ty) = tuple(elm.split('^^'))
                return Literal(value.replace('"', ''), datatype=URIRef(ty.lstrip('<').rstrip('>')))
            elif elm.startswith('_:'):
                return BNode(elm.replace('_:', ''))
            else:
                return Literal(elm.replace('"', ''), datatype=XSD.string)

        c, s, p, o = eval(x)
        return c, __term(s), __term(p), __term(o)

    for x in r.zrangebyscore('fragments:{}:stream'.format(fid), '-inf', '{}'.format(float(until))):
        yield __triplify(x)


def add_stream_triple(fid, tp, (s, p, o), timestamp=None):
    if timestamp is None:
        timestamp = calendar.timegm(dt.utcnow().timetuple())
    quad = (tp, s.n3(), p.n3(), o.n3())
    stream_key = 'fragments:{}:stream'.format(fid)
    not_found = not bool(r.zscore(stream_key, quad))
    if not_found:
        with r.pipeline() as pipe:
            pipe.zadd(stream_key, timestamp, quad)
            pipe.execute()
{'fraction_non_alphanumeric': 0.22180451127819548, 'fraction_numerical': 0.004511278195488722, 'mean_word_length': 2.700278164116829, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2618207', 'n_tokens_mistral': 1792, 'n_tokens_neox': 1600, 'n_words': 449}