<commit_before>'''
Created on 9 Dec 2014

@author: <NAME>
'''
from rdflib import Graph
from rdflib_appengine.ndbstore import NDBStore
from rdflib.plugins.sparql import prepareQuery
from google.appengine.api import search

_DOCUMENTS = '''
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> 
PREFIX sepake: <http://dot.rural/sepake/>
SELECT *
WHERE {
    ?sepakeuri rdfs:label ?label .
    OPTIONAL { ?sepakeuri sepake:htmlDescription ?htmlDescription } .
}
'''

def _document_from_sparql_result(sparql_result):
    fields = [search.HtmlField(name='label', value=sparql_result['label'])]
    if 'htmlDescription' in sparql_result:
        fields.append(search.HtmlField(name='description', value=sparql_result['htmlDescription']))
    return search.Document(doc_id = sparql_result['sepakeuri'], 
                           fields = fields,
                           rank = len(fields))
    
class Indexer(object):
    def __init__(self, graphid):
        self._index = search.Index(name = graphid)
        graph = Graph(store = NDBStore(identifier = graphid))
        query = prepareQuery(_DOCUMENTS)
        self._docs = graph.query(query)
        self._more = True

    def __iter__(self):
        while self._more:
            index_now = list()
            self._more = False
            for doc in self._docs:
                index_now.append(doc)
                if len(index_now) == 200:
                    self._more = True
                    break
            documents = [_document_from_sparql_result(doc.asdict()) for doc in index_now]
            self._index.put(documents)
            yield len(index_now)

<commit_msg>Indexing by rank For #42-rank<commit_after>'''
Created on 9 Dec 2014

@author: <NAME>
'''
from rdflib import Graph
from rdflib_appengine.ndbstore import NDBStore
from rdflib.plugins.sparql import prepareQuery
from google.appengine.api import search

_DOCUMENTS = '''
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> 
PREFIX sepake: <http://dot.rural/sepake/>
PREFIX sepakemetrics: <http://dot.rural/sepake/metrics/>
SELECT ?sepakeuri ?label ?htmlDescription (COUNT(*) AS ?rank)
WHERE {
    ?sepakeuri rdfs:label ?label .
    OPTIONAL { ?sepakeuri sepake:htmlDescription ?htmlDescription } .
    OPTIONAL { ?sepakeuri sepakemetrics:focushit []} . 
}
GROUP BY ?sepakeuri
'''

def _document_from_sparql_result(sparql_result):
    fields = [search.HtmlField(name='label', value=sparql_result['label'])]
    if 'htmlDescription' in sparql_result:
        fields.append(search.HtmlField(name='description', value=sparql_result['htmlDescription']))
    return search.Document(doc_id = sparql_result['sepakeuri'], 
                           fields = fields,
                           rank = sparql_result['rank'].value)
    
class Indexer(object):
    def __init__(self, graphid):
        self._index = search.Index(name = graphid)
        graph = Graph(store = NDBStore(identifier = graphid))
        query = prepareQuery(_DOCUMENTS)
        self._docs = graph.query(query)
        self._more = True

    def __iter__(self):
        while self._more:
            index_now = list()
            self._more = False
            for doc in self._docs:
                index_now.append(doc)
                if len(index_now) == 200:
                    self._more = True
                    break
            documents = [_document_from_sparql_result(doc.asdict()) for doc in index_now]
            self._index.put(documents)
            yield len(index_now)


{'fraction_non_alphanumeric': 0.10974212034383954, 'fraction_numerical': 0.009169054441260744, 'mean_word_length': 2.628898128898129, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6711666', 'n_tokens_mistral': 1114, 'n_tokens_neox': 1057, 'n_words': 227}