<commit_before>#!/usr/bin/env python3
import sys
import time
import json
import uuid
import collections
from urllib.request import FancyURLopener
from urllib.parse import urljoin, quote, urlencode, unquote, urlparse

import yaml

from elasticsearch import Elasticsearch
from elasticsearch.helpers import bulk
from elasticsearch.exceptions import NotFoundError

try:
    import ijson.backends.yajl2 as ijson
except:
    print("Can't find py-yajl, JSON parsing will be slower")
    import ijson

REPORT_EVERY=1200
DEFAULT_SPARQL_TIMEOUT=60

# SPARQL variable name for the resource
ID="id"
# and for the rdf:type (on subclasses)
TYPE="type"

class SlidingWindowDictionary(collections.OrderedDict):
    def __init__(self, max_size=1000):
        self.max_size = max_size
        super().__init__()
    def __setitem__(self, key, value):
        while (len(self) >= self.max_size):
            self.popitem(last=False)
        super().__setitem__(key, value)

def negate(f):
    return lambda *args,**kwargs: not f(*args, **kwargs) 

def is_property_required(prop):
    is_required = bool(prop.get("required", False))
    return is_required

class Session:
    def __init__(self, config_file):
        with open(config_file) as f:
            self.conf = yaml.load(f)

        es_hosts = self.conf.get("elasticsearch")
        print("###")
        print("# SPARQL endpoint: " + self.conf["sparql"]["uri"])
        print("# ElasticSearch: %s" % es_hosts)
        print("###")
        self.es = Elasticsearch(es_hosts)

        self.urlOpener = FancyURLopener()
        self.urlOpener.addheader("Accept",
            "application/sparql-results+json, applicaton/json;q=0.1")

    def uri_to_qname(self,uri):
        for p,u in self.conf.get("prefixes", {}).items():
            if uri.startswith(u):
                return uri.replace(u, p+":", 1)
        return uri

    def sparql_prefixes(self):
        sparql=[]
        for p,u in self.conf.get("prefixes", {}).items():
            sparql.append("PREFIX %s: <%s>" % (p,u))
        return "\n".join(sparql)

    def run(self):
        for index in self.conf["indexes"]:
            try:
                self.es.indices.delete(index=index)
            except NotFoundError:
                pass
            for doc_type in self.conf["indexes"][index]:
                indexer = Indexer(self, index, doc_type)
                ## TODO: Store mapping for JSON-LD
                indexer.load()

    def dryrun(self):
        for index in self.conf["indexes"]:
            for doc_type in self.conf["indexes"][index]:
                print("## index/type:", index, doc_type)
                indexer = Indexer(self, index, doc_type)
                # below should print the sparql
                indexer.sparql()

    def check(self):
        self.check_prefixes()
        self.check_required_properties()

    def expand_qname(self, p):
        if not ":" in p:
            raise Exception("Invalid property, no prefix: " + p)
        prefix,rest = p.split(":", 1)
        if not prefix in self.conf.get("prefixes", {}):
            raise Exception("Unknown prefix: " + prefix)
        base = self.conf.get("prefixes")[prefix]
        return base + rest

    def check_property(self, p):
        if type(p) == str:
            urlparse(self.expand_qname(p))
        else:
            ## Assume it is dict-based - check they are all non-empty
            if not p.get("sparql"):
                raise Exception("'sparql' missing for %s" % p)
            if not p.get("variable"):
                raise Exception("'variable' missing for %s" % p)
            if not p.get("jsonld"):
                raise Exception("'jsonld' missing for %s" % p)

    def check_required_properties(self):
        # Check that every index+type have at least one
        # required triple (rdf:type or a property)

        for p in self.conf["common_properties"]:
            if type(p) != str and is_property_required(p):
                return # Great! required for every index

        # if not, we'll need to check each index+type
        for index,index_conf in self.conf["indexes"].items():
            for doc_type,type_conf in index_conf.items():
                if "type" in type_conf:
                    continue # OK
                if filter(is_property_required, type_conf.get("properties", [])):
                    continue # OK
                raise Exception("No type: or property with required:true for %s %s" % (index, doc_type))

    def check_prefixes(self):
        for uri in self.conf.get("prefixes", {}).values():
            if not (uri.endswith("#") or uri.endswith("/")):
                # This should catch prefix definitions not ending with / #
                print("WARNING: Prefix doesn't end with / or #: %s" % uri,
                    file=sys.stderr)
        for p in self.conf.get("common_properties", []):
            self.check_property(p)
        for index,index_conf in self.conf["indexes"].items():
            for doc_type,type_conf in index_conf.items():
                if "type" in type_conf:
                    urlparse(self.expand_qname(type_conf["type"]))
                for p in type_conf.get("properties", []):
                    self.check_property(p)
                    



class Indexer:
    def __init__(self, session, index, doc_type):
        self.cache = SlidingWindowDictionary()
        self.session = session<commit_msg>support missing common_properties<commit_after>#!/usr/bin/env python3
import sys
import time
import json
import uuid
import collections
from urllib.request import FancyURLopener
from urllib.parse import urljoin, quote, urlencode, unquote, urlparse

import yaml

from elasticsearch import Elasticsearch
from elasticsearch.helpers import bulk
from elasticsearch.exceptions import NotFoundError

try:
    import ijson.backends.yajl2 as ijson
except:
    print("Can't find py-yajl, JSON parsing will be slower")
    import ijson

REPORT_EVERY=1200
DEFAULT_SPARQL_TIMEOUT=60

# SPARQL variable name for the resource
ID="id"
# and for the rdf:type (on subclasses)
TYPE="type"

class SlidingWindowDictionary(collections.OrderedDict):
    def __init__(self, max_size=1000):
        self.max_size = max_size
        super().__init__()
    def __setitem__(self, key, value):
        while (len(self) >= self.max_size):
            self.popitem(last=False)
        super().__setitem__(key, value)

def negate(f):
    return lambda *args,**kwargs: not f(*args, **kwargs)

def is_property_required(prop):
    is_required = bool(prop.get("required", False))
    return is_required

class Session:
    def __init__(self, config_file):
        with open(config_file) as f:
            self.conf = yaml.load(f)

        es_hosts = self.conf.get("elasticsearch")
        print("###")
        print("# SPARQL endpoint: " + self.conf["sparql"]["uri"])
        print("# ElasticSearch: %s" % es_hosts)
        print("###")
        self.es = Elasticsearch(es_hosts)

        self.urlOpener = FancyURLopener()
        self.urlOpener.addheader("Accept",
            "application/sparql-results+json, applicaton/json;q=0.1")

    def uri_to_qname(self,uri):
        for p,u in self.conf.get("prefixes", {}).items():
            if uri.startswith(u):
                return uri.replace(u, p+":", 1)
        return uri

    def sparql_prefixes(self):
        sparql=[]
        for p,u in self.conf.get("prefixes", {}).items():
            sparql.append("PREFIX %s: <%s>" % (p,u))
        return "\n".join(sparql)

    def run(self):
        for index in self.conf["indexes"]:
            try:
                self.es.indices.delete(index=index)
            except NotFoundError:
                pass
            for doc_type in self.conf["indexes"][index]:
                indexer = Indexer(self, index, doc_type)
                ## TODO: Store mapping for JSON-LD
                indexer.load()

    def dryrun(self):
        for index in self.conf["indexes"]:
            for doc_type in self.conf["indexes"][index]:
                print("## index/type:", index, doc_type)
                indexer = Indexer(self, index, doc_type)
                # below should print the sparql
                indexer.sparql()

    def check(self):
        self.check_prefixes()
        self.check_required_properties()

    def expand_qname(self, p):
        if not ":" in p:
            raise Exception("Invalid property, no prefix: " + p)
        prefix,rest = p.split(":", 1)
        if not prefix in self.conf.get("prefixes", {}):
            raise Exception("Unknown prefix: " + prefix)
        base = self.conf.get("prefixes")[prefix]
        return base + rest

    def check_property(self, p):
        if type(p) == str:
            urlparse(self.expand_qname(p))
        else:
            ## Assume it is dict-based - check they are all non-empty
            if not p.get("sparql"):
                raise Exception("'sparql' missing for %s" % p)
            if not p.get("variable"):
                raise Exception("'variable' missing for %s" % p)
            if not p.get("jsonld"):
                raise Exception("'jsonld' missing for %s" % p)

    def check_required_properties(self):
        # Check that every index+type have at least one
        # required triple (rdf:type or a property)

        for p in self.conf.get("common_properties", []):
            if type(p) != str and is_property_required(p):
                return # Great! required for every index

        # if not, we'll need to check each index+type
        for index,index_conf in self.conf["indexes"].items():
            for doc_type,type_conf in index_conf.items():
                if "type" in type_conf:
                    continue # OK
                if filter(is_property_required, type_conf.get("properties", [])):
                    continue # OK
                raise Exception("No type: or property with required:true for %s %s" % (index, doc_type))

    def check_prefixes(self):
        for uri in self.conf.get("prefixes", {}).values():
            if not (uri.endswith("#") or uri.endswith("/")):
                # This should catch prefix definitions not ending with / #
                print("WARNING: Prefix doesn't end with / or #: %s" % uri,
                    file=sys.stderr)
        for p in self.conf.get("common_properties", []):
            self.check_property(p)
        for index,index_conf in self.conf["indexes"].items():
            for doc_type,type_conf in index_conf.items():
                if "type" in type_conf:
                    urlparse(self.expand_qname(type_conf["type"]))
                for p in type_conf.get("properties", []):
                    self.check_property(p)




class Indexer:
    def __init__(self, session, index, doc_type):
        self.cache = SlidingWindowDictionary()
        self.session = session
{'fraction_non_alphanumeric': 0.11828851307642547, 'fraction_numerical': 0.0029572128269106366, 'mean_word_length': 2.2246722288438616, 'pattern_counts': {'":': 6, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '45851', 'n_tokens_mistral': 3259, 'n_tokens_neox': 3153, 'n_words': 845}