<commit_before>import re

class QueryBuilder(object):
    def __init__(self, root_type, root_name):
        self.root_type = root_type
        self.triples_created = []
        self.identifier_count = {}
        
        self.root_name = self.get_identifier(root_name)

    def construct_triples(self):
        return "\n ".join(self.triples_created)
        
    def require_above(self, above_type=None, above_uri=None):
        if (above_uri == None): return
        predicate = above_type.predicate_for_contained_type(self.root_type)
        predicate = str(predicate.uri)
        self.required_triple("<"+above_uri+">", "<"+predicate+">", self.root_name )
        
    def get_identifier(self, id_base, role="predicate"):
        if id_base[0] == "<" or id_base.startswith("_:"): return id_base
        
        start = id_base[0] == "?" and "?" or ""

        if "/" in id_base:
          id_base = id_base.rsplit("/", 1)[1]
        if "#" in id_base:
          id_base = id_base.rsplit("#", 1)[1]

        id_base = re.sub(r'\W+', '', id_base)
        id_base = start + id_base + "_" + role

        self.identifier_count.setdefault(id_base, 0)
        self.identifier_count[id_base] += 1
        return "%s_%s"%(id_base, self.identifier_count[id_base])

    def required_triple(self, root_name, pred, obj):
        self.triples_created.append("%s %s %s. " % (root_name, pred, obj))
        return " %s %s %s. \n" % (root_name, pred, obj)

    def optional_triple(self, root_name, pred, obj):
        self.triples_created.append("%s %s %s. " % (root_name, pred, obj))
        return " OPTIONAL { %s %s %s. } \n" % (root_name, pred, obj)
        
    def optional_linked_type(self, linked_type, root_name,  predicate, object, to_follow, depth):
        self.triples_created.append("%s %s %s. " % (root_name, predicate, object))
        ret = " OPTIONAL { %s %s %s. $insertion } \n" % (root_name, predicate, object)
        repl = self.build(to_follow, linked_type, depth)
        ret = ret.replace("$insertion", repl)
        return ret

    def build(self, root_name=None, root_type=None, depth=0):
        ret = ""
        # Recursion starting off:  set initial conditions (if any).
        if root_type == None:
            root_name = self.root_name
            root_type = self.root_type            
            ret = " ".join(self.triples_created)
            
        if (root_type.base_path != None):
            ret += self.required_triple(root_name, "rdf:type", "<"+str(root_type.node.uri)+">")
        else:
            ret += self.optional_triple(root_name, "rdf:type", "<"+str(root_type.node.uri)+">")
<commit_msg>Updated smart_common<commit_after>import re

class QueryBuilder(object):
    def __init__(self, root_type, root_name):
        self.root_type = root_type
        self.triples_created = []
        self.identifier_count = {}
        self.root_name = self.get_identifier(root_name)

    def construct_triples(self):
        return "\n ".join(self.triples_created)
        
    def require_above(self, above_type=None, above_uri=None):
        if (above_uri == None): return
        predicate = above_type.predicate_for_contained_type(self.root_type)
        predicate = str(predicate.uri)
        self.required_triple("<"+above_uri+">", "<"+predicate+">", self.root_name )
        
    def get_identifier(self, id_base, role=""):
        if id_base[0] == "<" or id_base.startswith("_:"): return id_base
        
        start = id_base[0] == "?" and "?" or ""

        if "/" in id_base:
          id_base = id_base.rsplit("/", 1)[1]
        if "#" in id_base:
          id_base = id_base.rsplit("#", 1)[1]

        id_base = re.sub(r'\W+', '', id_base)
        id_base = start + id_base
        if role != "":
            id_base = id_base + "_" + role

        self.identifier_count.setdefault(id_base, 0)
        self.identifier_count[id_base] += 1
        ret = "%s"%id_base
        c = self.identifier_count[id_base]
        if c > 1:
            ret += "_%s"%c
        return ret

    def required_triple(self, root_name, pred, obj):
        self.triples_created.append("%s %s %s. " % (root_name, pred, obj))
        return " %s %s %s. \n" % (root_name, pred, obj)

    def optional_triple(self, root_name, pred, obj):
        self.triples_created.append("%s %s %s. " % (root_name, pred, obj))
        return " OPTIONAL { %s %s %s. } \n" % (root_name, pred, obj)
        
    def optional_linked_type(self, linked_type, root_name,  predicate, object, to_follow, depth):
        self.triples_created.append("%s %s %s. " % (root_name, predicate, object))
        ret = " OPTIONAL { %s %s %s. $insertion } \n" % (root_name, predicate, object)
        repl = self.build(to_follow, linked_type, depth)
        ret = ret.replace("$insertion", repl)
        return ret

    def build(self, root_name=None, root_type=None, depth=0):
        ret = ""
        
        # Recursion starting off:  set initial conditions (if any).
        if root_type == None:
            root_name = self.root_name
            root_type = self.root_type            
            ret = " ".join(self.triples_created)

        # If there's a type, it must be the root_type
        type_id = self.get_identifier("?rdftype", "object")
        ret += self.optional_triple(root_name, "rdf:type", type_id)  + \
               "FILTER (!BOUND(%s) || %s = <%s>) "% (type_id, type_id, str(root_type.node.uri))

{'fraction_non_alphanumeric': 0.14425744405400406, 'fraction_numerical': 0.0035139633808026634, 'mean_word_length': 2.464445868033312, 'pattern_counts': {'":': 1, '<': 12, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '112441', 'n_tokens_mistral': 1849, 'n_tokens_neox': 1816, 'n_words': 386}