<commit_before>"""Provide an interface to SPARQL query endpoints."""

import urllib
from cStringIO import StringIO

import simplejson
import httplib2
import rdflib
from lxml import objectify

import logging

log = logging.getLogger(__name__)

class SPARQLQueryException(Exception):
    """Raised when the SPARQL store returns an HTTP status code other than 200 OK."""
    pass

class UnknownSPARQLReturnTypeException(Exception):
    """Raised when the SPARQL store provides a response with an unrecognized content-type."""
    pass

class SPARQLServer(object):
    """A server that can run SPARQL queries."""
    
    def __init__(self, query_url):
        self.query_url = query_url
        
    def query(self, sparql, output='json'):
        """Executes a SPARQL query. The return type varies based on what the
        SPARQL store responds with:
        
        * application/rdf+xml: an rdflib.ConjunctiveGraph
        * application/sparql-results+json: A dictionary from simplejson
        * application/sparql-results+xml: An lxml.objectify structure
        
        :param sparql: The SPARQL to execute.
        :returns: The results of the query from the SPARQL store."""
        http = httplib2.Http()
        
        log.debug("Querying: %s with: %r", self.query_url, sparql)
        response, content = http.request(
            uri=self.query_url, method='POST',
            body=urllib.urlencode({'query': sparql, 'output':output }), 
            headers={
                "Content-Type": "application/x-www-form-urlencoded",
            })
        if response['status'] != '200':
            raise SPARQLQueryException('%s: %s' % (response, content))
        if response['content-type'].startswith('application/rdf+xml'):
            graph = rdflib.ConjunctiveGraph()
            graph.parse(StringIO(content), self.query_url)
            return graph
        elif response['content-type'].startswith('application/sparql-results+json'):
            return simplejson.loads(content)
        elif response['content-type'].startswith('application/sparql-results+xml'):
            return objectify.parse(StringIO(content))
        else:
            raise UnknownSPARQLReturnTypeException('Got content of type: %s' %\
                                                   response['content-type'])

class UpdateableGraphStore(SPARQLServer):
    """SPARQL server class that is capable of interacting with SPARQL 1.1
    graph stores."""
    
    def __init__(self, query_url, dataset_url):
        super(UpdateableRDFStore, self).__init__(query_url)
        self.dataset_url = dataset_url
    
    def request_url(self, graph_uri):
        return urllib.urlencode({'graph': graph_uri})
    
    def get(self, graph_uri):
        h = httplib2.Http()
        resp, content = h.request(uri = self.request_url(graph_uri), method = 'GET')
        if resp['status'] != '200':
            raise Exception('Error from Graph Store (%s): %s' %\
                            (resp['status'], content))
        graph = rdflib.ConjunctiveGraph()
        if resp['content-type'].startswith('application/rdf+xml'):
            graph.parse(StringIO(content), publicID=graph_uri, format='xml')
        elif resp['content-type'].startswith('text/turtle'):
            graph.parse(StringIO(content), publicID=graph_uri, format='turtle')
        elif resp['content-type'].startswith('text/rdf+n3'):
            graph.parse(StringIO(content), publicID=graph_uri, format='n3')
        elif resp['content-type'].startswith('text/plain'):
            graph.parse(StringIO(content), publiCID=graph_uri, format='nt')
        return graph
    
    def delete(self, graph_uri):
        h = httplib2.Http()<commit_msg>Send proper accept headers on GET requests.<commit_after>"""Provide an interface to SPARQL query endpoints."""

import urllib
from cStringIO import StringIO

import simplejson
import httplib2
import rdflib
from lxml import objectify

import logging

log = logging.getLogger(__name__)

class SPARQLQueryException(Exception):
    """Raised when the SPARQL store returns an HTTP status code other than 200 OK."""
    pass

class UnknownSPARQLReturnTypeException(Exception):
    """Raised when the SPARQL store provides a response with an unrecognized content-type."""
    pass

class SPARQLServer(object):
    """A server that can run SPARQL queries."""
    
    def __init__(self, query_url):
        self.query_url = query_url
        
    def query(self, sparql, output='json'):
        """Executes a SPARQL query. The return type varies based on what the
        SPARQL store responds with:
        
        * application/rdf+xml: an rdflib.ConjunctiveGraph
        * application/sparql-results+json: A dictionary from simplejson
        * application/sparql-results+xml: An lxml.objectify structure
        
        :param sparql: The SPARQL to execute.
        :returns: The results of the query from the SPARQL store."""
        http = httplib2.Http()
        
        log.debug("Querying: %s with: %r", self.query_url, sparql)
        response, content = http.request(
            uri=self.query_url, method='POST',
            body=urllib.urlencode({'query': sparql, 'output':output }), 
            headers={
                "Content-Type": "application/x-www-form-urlencoded",
            })
        if response['status'] != '200':
            raise SPARQLQueryException('%s: %s' % (response, content))
        if response['content-type'].startswith('application/rdf+xml'):
            graph = rdflib.ConjunctiveGraph()
            graph.parse(StringIO(content), self.query_url)
            return graph
        elif response['content-type'].startswith('application/sparql-results+json'):
            return simplejson.loads(content)
        elif response['content-type'].startswith('application/sparql-results+xml'):
            return objectify.parse(StringIO(content))
        else:
            raise UnknownSPARQLReturnTypeException('Got content of type: %s' %\
                                                   response['content-type'])

class UpdateableGraphStore(SPARQLServer):
    """SPARQL server class that is capable of interacting with SPARQL 1.1
    graph stores."""
    
    def __init__(self, query_url, dataset_url):
        super(UpdateableRDFStore, self).__init__(query_url)
        self.dataset_url = dataset_url
    
    def request_url(self, graph_uri):
        return urllib.urlencode({'graph': graph_uri})
    
    def get(self, graph_uri):
        h = httplib2.Http()
        resp, content = h.request(
            uri = self.request_url(graph_uri), method = 'GET',
            headers = {'Accept': 'text/plain,application/rdf+xml,text/turtle,text/rdf+n3',},)
        if resp['status'] != '200':
            raise Exception('Error from Graph Store (%s): %s' %\
                            (resp['status'], content))
        graph = rdflib.ConjunctiveGraph()
        if resp['content-type'].startswith('text/plain'):
            graph.parse(StringIO(content), publiCID=graph_uri, format='nt')
        elif resp['content-type'].startswith('application/rdf+xml'):
            graph.parse(StringIO(content), publicID=graph_uri, format='xml')
        elif resp['content-type'].startswith('text/turtle'):
            graph.parse(StringIO(content), publicID=graph_uri, format='turtle')
        elif resp['content-type'].startswith('text/rdf+n3'):
            graph.parse(StringIO(content), publicID=graph_uri, format='n3')
        return graph
    
    def delete(self, graph_uri):
        h = httplib2.Http()
{'fraction_non_alphanumeric': 0.11668657905217045, 'fraction_numerical': 0.004646223284216116, 'mean_word_length': 2.7352503718393653, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 17}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6050099', 'n_tokens_mistral': 2190, 'n_tokens_neox': 2138, 'n_words': 515}