<commit_before>'''
Created on 19 Aug 2011

@author: hoekstra
'''
from whoosh.index import create_in, open_dir
from whoosh.fields import ID, DATETIME, TEXT, Schema
from whoosh.query import *
from whoosh.qparser import QueryParser
import os.path
from xml.etree.ElementTree import ElementTree
import glob
from SPARQLWrapper import SPARQLWrapper, JSON
from datetime import datetime

SPARQL_ENDPOINT = "http://doc.metalex.eu:8000/sparql/"
INDEX_DIR = "/var/metalex/store/index"
#INDEX_DIR = "index"
DOCS_DIR = "/var/metalex/store/data"
#DOCS_DIR = ".."

tree = ElementTree()

filelist = glob.glob("{}/*_ml.xml".format(DOCS_DIR))


schema = Schema(uri=ID(stored=True),valid=DATETIME(stored=True),title=TEXT,ctitle=TEXT)

# should become /var/metalex/store/index
if not os.path.exists(INDEX_DIR) :
    os.mkdir(INDEX_DIR)
    ix = create_in(INDEX_DIR, schema)
else :
    ix = open_dir(INDEX_DIR)

writer = ix.writer()
searcher = ix.searcher()

print list(searcher.lexicon("uri"))

for f in filelist :
    print f
    try :
        file_root = tree.parse(f)
        uri = file_root.attrib['about']
        
        wq = Term("uri",uri.decode('utf-8'))
        
        results = searcher.search(wq)
        
        if len(results) > 1 :
            print "Document already indexed"
        else :
            q = """PREFIX dcterms: <http://purl.org/dc/terms/> 
                PREFIX metalex: <http://www.metalex.eu/schema/1.0#> 
                PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
                PREFIX foaf: <http://xmlns.com/foaf/0.1/>
                PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> 
                
                SELECT DISTINCT ?title ?date ?ctitle WHERE {
                   <"""+uri+"""> a metalex:BibliographicExpression .
                   <"""+uri+"""> dcterms:valid ?date .
                   <"""+uri+"""> dcterms:title ?title .
                   OPTIONAL { <"""+uri+"""> dcterms:alternative ?ctitle . }
                } """
                
    
            sparql = SPARQLWrapper(SPARQL_ENDPOINT)
            sparql.setQuery(q)
            
            sparql.setReturnFormat(JSON)
            sparql_results = sparql.query().convert()
            
            
            for row in sparql_results['results']['bindings'] :
                print row
                title = row['title']['value']
                valid = row['date']['value']
                
                valid_date = datetime.strptime(valid,'%Y-%m-%d')
                
                if 'ctitle' in row :
                    ctitle = row['ctitle']['value']
                    writer.add_document(uri=uri.decode('utf-8'),title=title.decode('utf-8'), ctitle=ctitle.decode('utf-8'), valid=valid_date)
                else :
                    writer.add_document(uri=uri.decode('utf-8'),title=title.decode('utf-8'), valid=valid_date)
                    ctitle = ''
                
                
                print uri, title, ctitle, valid
            
        
    except Exception as e:
        print "Some error: {}".format(e)
        print e
        
writer.commit()
<commit_msg>Fixed check for citation title<commit_after>'''
Created on 19 Aug 2011

@author: hoekstra
'''
from whoosh.index import create_in, open_dir
from whoosh.fields import ID, DATETIME, TEXT, Schema
from whoosh.query import *
from whoosh.qparser import QueryParser
import os.path
from xml.etree.ElementTree import ElementTree
import glob
from SPARQLWrapper import SPARQLWrapper, JSON
from datetime import datetime

SPARQL_ENDPOINT = "http://doc.metalex.eu:8000/sparql/"
INDEX_DIR = "/var/metalex/store/index"
#INDEX_DIR = "index"
DOCS_DIR = "/var/metalex/store/data"
#DOCS_DIR = ".."

tree = ElementTree()

filelist = glob.glob("{}/*_ml.xml".format(DOCS_DIR))


schema = Schema(uri=ID(stored=True),valid=DATETIME(stored=True),title=TEXT,ctitle=TEXT)

# should become /var/metalex/store/index
if not os.path.exists(INDEX_DIR) :
    os.mkdir(INDEX_DIR)
    ix = create_in(INDEX_DIR, schema)
else :
    ix = open_dir(INDEX_DIR)

writer = ix.writer()
searcher = ix.searcher()

print list(searcher.lexicon("uri"))

for f in filelist :
    print f
    try :
        file_root = tree.parse(f)
        uri = file_root.attrib['about']
        
        wq = Term("uri",uri.decode('utf-8'))
        
        results = searcher.search(wq)
        
        if len(results) > 1 :
            print "Document already indexed"
        else :
            q = """PREFIX dcterms: <http://purl.org/dc/terms/> 
                PREFIX metalex: <http://www.metalex.eu/schema/1.0#> 
                PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
                PREFIX foaf: <http://xmlns.com/foaf/0.1/>
                PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> 
                
                SELECT DISTINCT ?title ?date ?ctitle WHERE {
                   <"""+uri+"""> a metalex:BibliographicExpression .
                   <"""+uri+"""> dcterms:valid ?date .
                   <"""+uri+"""> dcterms:title ?title .
                   OPTIONAL { <"""+uri+"""> dcterms:alternative ?ctitle . }
                } """
                
    
            sparql = SPARQLWrapper(SPARQL_ENDPOINT)
            sparql.setQuery(q)
            
            sparql.setReturnFormat(JSON)
            sparql_results = sparql.query().convert()
            
            
            for row in sparql_results['results']['bindings'] :
                print row
                title = row['title']['value']
                valid = row['date']['value']
                
                valid_date = datetime.strptime(valid,'%Y-%m-%d')
                
                if 'value' in row['ctitle'] :
                    ctitle = row['ctitle']['value']
                    writer.add_document(uri=uri.decode('utf-8'),title=title.decode('utf-8'), ctitle=ctitle.decode('utf-8'), valid=valid_date)
                else :
                    writer.add_document(uri=uri.decode('utf-8'),title=title.decode('utf-8'), valid=valid_date)
                    ctitle = ''
                
                
                print uri, title, ctitle, valid
            
        
    except Exception as e:
        print "Some error: {}".format(e)
        print e
        
writer.commit()

{'fraction_non_alphanumeric': 0.13476312419974393, 'fraction_numerical': 0.01056338028169014, 'mean_word_length': 1.8731034482758622, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1943206', 'n_tokens_mistral': 2053, 'n_tokens_neox': 1927, 'n_words': 370}