<commit_before>import csv
import rdflib
from rdflib.namespace import RDFS, RDF, OWL
from rdflib.term import URIRef
import threading

from apimarkdown import Markdown
from apirdflib import RDFLIBLOCK

import logging
logging.basicConfig(level=logging.INFO) # dev_appserver.py --log_level debug .
log = logging.getLogger(__name__)

class sdordf2csv():
    
    def __init__(self, queryGraph=None, fullGraph=None, markdownComments=True,excludeAttic=False):
        self.setqueryGraph(queryGraph) 
        self.setfullGraph(fullGraph)
        self.setexcludeAttic(excludeAttic)
        self.setmarkdownComments(markdownComments)
    
    def setqueryGraph(self,graph=None):
        self.queryGraph = graph
    
    def setfullGraph(self,graph=None):
        self.fullGraph = graph
        
    def setexcludeAttic(self,state):
        self.excludeAttic = state
        
    def setmarkdownComments(self,state):
        self.markdown = state
        
    def doQuery(self,graph=None,query=None):
        res = None
        try:
            RDFLIBLOCK.acquire()
            res = list(graph.query(query))
        finally:
            RDFLIBLOCK.release()
        return res    

    def outputCSVtypes(self,file):
        atticfilter = ""
        if self.excludeAttic:
            atticfilter = "FILTER NOT EXISTS {?term schema:isPartOf <http://attic.schema.org>}."
        query= ('''select ?term where { 
           ?term a rdfs:Class.
           BIND(STR(?term) AS ?strVal)
           FILTER(STRLEN(?strVal) >= 18 && SUBSTR(?strVal, 1, 18) = "http://schema.org/")
           %s
        }
        ORDER BY ?term
         ''') % atticfilter
        try:
            RDFLIBLOCK.acquire()
            types = list(self.queryGraph.query(query))
        finally:
            RDFLIBLOCK.release()
        #log.info( "Types: %s" % len(types))
        self.type2CSV(header=True,out=file)
        for t in types:
            self.type2CSV(term=t.term,header=False,out=file,graph=self.queryGraph)
        
    def outputCSVproperties(self,file):
        atticfilter = ""
        if self.excludeAttic:
            atticfilter = "FILTER NOT EXISTS {?term schema:isPartOf <http://attic.schema.org>}."
        query= ('''select ?term where { 
           ?term a rdf:Property.
           FILTER EXISTS {?term rdfs:label ?l}.
           BIND(STR(?term) AS ?strVal).
           FILTER(STRLEN(?strVal) >= 18 && SUBSTR(?strVal, 1, 18) = "http://schema.org/").
           %s           
        }
        ORDER BY ?term''') % atticfilter
        props = list(self.queryGraph.query(query))
        self.prop2CSV(header=True,out=file)
        for t in props:
            self.prop2CSV(term=t.term,header=False,out=file,graph=self.queryGraph)

    def outputCSVenums(self,file):
        atticfilter = ""
        if self.excludeAttic:
            atticfilter = "FILTER NOT EXISTS {?term schema:isPartOf <http://attic.schema.org>.}"
        query= ('''select ?term where{
         ?term a ?type. 
         ?type rdfs:subClassOf* <http://schema.org/Enumeration>.
         %s 
        }
        ORDER BY ?term''') % atticfilter
        allenums = list(self.fullGraph.query(query))

        query= ('''select DISTINCT ?term where{
         ?term ?p ?o. 
         FILTER NOT EXISTS {?term a rdfs:Class}. 
         FILTER NOT EXISTS {?term a rdf:Property}.
         %s 
        }
        ORDER BY ?term''') % atticfilter
        terms = list(self.queryGraph.query(query))
        enums = []
        for t in terms:
            if t in allenums:
                enums.append(t)
        self.enum2CSV(header=True,out=file)
        for t in enums:
            self.enum2CSV(term=t.term,header=False,out=file,graph=self.queryGraph)
        
    def enum2CSV(self,term=None,header=True,out=None,graph=None):
        cols = ["id","label","comment","enumerationtype","supersedes","supersededBy","isPartOf"]
        if not out:
            return
        writer = csv.writer(out,quoting=csv.QUOTE_ALL,lineterminator='\n')
        if header:
            writer.writerow(cols)
            return
        if not graph:
            graph = self.queryGraph
        if term == None or graph == None:
            return
        row = [str(term)]
        row.append(self.graphValueToCSV(subject=term,predicate=RDFS.label,graph=graph))
        row.append(self.getCSVComment(term,graph=self.fullGraph))
        row.append(self.graphValueToCSV(subject=term,predicate=RDF.type,graph=graph))
        row.append(self.getCSVsuperseds(term,graph=self.fullGraph))
        row.append(self.getCSVSupersededBy(term,graph=self.fullGraph))
        row.append(self.graphValueToCSV(subject=term,predicate=URIRef("http://schema.org/isPartOf"),graph=graph))

        row=[s.encode('utf-8') for s in row]
        writer.writerow(row)

    def prop2CSV(self,term=None,header=True,out=None,graph=None):
        cols = ["id","label","comment","subPropertyOf","equivalentProperty","subproperties","domainIncludes","rangeIncludes","inversOf","supersedes","supersededBy","isPartOf"]<commit_msg>Fix typo of csv column name for inverseOf Issue (#1834)<commit_after>import csv
import rdflib
from rdflib.namespace import RDFS, RDF, OWL
from rdflib.term import URIRef
import threading

from apimarkdown import Markdown
from apirdflib import RDFLIBLOCK

import logging
logging.basicConfig(level=logging.INFO) # dev_appserver.py --log_level debug .
log = logging.getLogger(__name__)

class sdordf2csv():
    
    def __init__(self, queryGraph=None, fullGraph=None, markdownComments=True,excludeAttic=False):
        self.setqueryGraph(queryGraph) 
        self.setfullGraph(fullGraph)
        self.setexcludeAttic(excludeAttic)
        self.setmarkdownComments(markdownComments)
    
    def setqueryGraph(self,graph=None):
        self.queryGraph = graph
    
    def setfullGraph(self,graph=None):
        self.fullGraph = graph
        
    def setexcludeAttic(self,state):
        self.excludeAttic = state
        
    def setmarkdownComments(self,state):
        self.markdown = state
        
    def doQuery(self,graph=None,query=None):
        res = None
        try:
            RDFLIBLOCK.acquire()
            res = list(graph.query(query))
        finally:
            RDFLIBLOCK.release()
        return res    

    def outputCSVtypes(self,file):
        atticfilter = ""
        if self.excludeAttic:
            atticfilter = "FILTER NOT EXISTS {?term schema:isPartOf <http://attic.schema.org>}."
        query= ('''select ?term where { 
           ?term a rdfs:Class.
           BIND(STR(?term) AS ?strVal)
           FILTER(STRLEN(?strVal) >= 18 && SUBSTR(?strVal, 1, 18) = "http://schema.org/")
           %s
        }
        ORDER BY ?term
         ''') % atticfilter
        try:
            RDFLIBLOCK.acquire()
            types = list(self.queryGraph.query(query))
        finally:
            RDFLIBLOCK.release()
        #log.info( "Types: %s" % len(types))
        self.type2CSV(header=True,out=file)
        for t in types:
            self.type2CSV(term=t.term,header=False,out=file,graph=self.queryGraph)
        
    def outputCSVproperties(self,file):
        atticfilter = ""
        if self.excludeAttic:
            atticfilter = "FILTER NOT EXISTS {?term schema:isPartOf <http://attic.schema.org>}."
        query= ('''select ?term where { 
           ?term a rdf:Property.
           FILTER EXISTS {?term rdfs:label ?l}.
           BIND(STR(?term) AS ?strVal).
           FILTER(STRLEN(?strVal) >= 18 && SUBSTR(?strVal, 1, 18) = "http://schema.org/").
           %s           
        }
        ORDER BY ?term''') % atticfilter
        props = list(self.queryGraph.query(query))
        self.prop2CSV(header=True,out=file)
        for t in props:
            self.prop2CSV(term=t.term,header=False,out=file,graph=self.queryGraph)

    def outputCSVenums(self,file):
        atticfilter = ""
        if self.excludeAttic:
            atticfilter = "FILTER NOT EXISTS {?term schema:isPartOf <http://attic.schema.org>.}"
        query= ('''select ?term where{
         ?term a ?type. 
         ?type rdfs:subClassOf* <http://schema.org/Enumeration>.
         %s 
        }
        ORDER BY ?term''') % atticfilter
        allenums = list(self.fullGraph.query(query))

        query= ('''select DISTINCT ?term where{
         ?term ?p ?o. 
         FILTER NOT EXISTS {?term a rdfs:Class}. 
         FILTER NOT EXISTS {?term a rdf:Property}.
         %s 
        }
        ORDER BY ?term''') % atticfilter
        terms = list(self.queryGraph.query(query))
        enums = []
        for t in terms:
            if t in allenums:
                enums.append(t)
        self.enum2CSV(header=True,out=file)
        for t in enums:
            self.enum2CSV(term=t.term,header=False,out=file,graph=self.queryGraph)
        
    def enum2CSV(self,term=None,header=True,out=None,graph=None):
        cols = ["id","label","comment","enumerationtype","supersedes","supersededBy","isPartOf"]
        if not out:
            return
        writer = csv.writer(out,quoting=csv.QUOTE_ALL,lineterminator='\n')
        if header:
            writer.writerow(cols)
            return
        if not graph:
            graph = self.queryGraph
        if term == None or graph == None:
            return
        row = [str(term)]
        row.append(self.graphValueToCSV(subject=term,predicate=RDFS.label,graph=graph))
        row.append(self.getCSVComment(term,graph=self.fullGraph))
        row.append(self.graphValueToCSV(subject=term,predicate=RDF.type,graph=graph))
        row.append(self.getCSVsuperseds(term,graph=self.fullGraph))
        row.append(self.getCSVSupersededBy(term,graph=self.fullGraph))
        row.append(self.graphValueToCSV(subject=term,predicate=URIRef("http://schema.org/isPartOf"),graph=graph))

        row=[s.encode('utf-8') for s in row]
        writer.writerow(row)

    def prop2CSV(self,term=None,header=True,out=None,graph=None):
        cols = ["id","label","comment","subPropertyOf","equivalentProperty","subproperties","domainIncludes","rangeIncludes","inverseOf","supersedes","supersededBy","isPartOf"]
{'fraction_non_alphanumeric': 0.12593033640964574, 'fraction_numerical': 0.00436637888260395, 'mean_word_length': 2.437244201909959, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5123439', 'n_tokens_mistral': 3331, 'n_tokens_neox': 3136, 'n_words': 572}