<commit_before># -*- coding: utf-8 -*-
import datetime
import sys
import operator
from pyretrieval import processor, document, indexer

start = datetime.datetime.now()
print "Initializing Document Processor"
proc = processor.Processor()

# LOAD STOPWORDS
print "Loading Stopwords..."
temp_time = datetime.datetime.now()
proc.load_stopwords('german_example_files/stopwords_german', True)
print "Finished after:", str(datetime.datetime.now()-temp_time)

# LOAD LEMMAS
print "Loading Wordlist..."
temp_time = datetime.datetime.now()
proc.load_lemmas('german_example_files/wordlist_german', True)
print "Finished after:", str(datetime.datetime.now()-temp_time)

# PROCESS DOCUMENTS
print "Processing Documents"
docs = []
temp_time = datetime.datetime.now()
with open('german_example_files/german_news_example') as file:
    i = 0
    for line in file:
        i += 1
        sys.stdout.write(str(i)+'\r')
        docs.append(proc.process(line.decode("utf-8")))
print "Finished after:", str(datetime.datetime.now()-temp_time)

# INDEX DOCUMENTS
print "Indexing Documents..."
idxr = indexer.Indexer()
temp_time = datetime.datetime.now()
for doc in docs:
    idxr.index(doc)
print "Finished after:", str(datetime.datetime.now()-temp_time)

# CALCULATE INVERSE DOCUMENT FREQUENCY
temp_time = datetime.datetime.now()
print "Calculating inverse document frequency"
idxr.calc_idfs()
print "Finished after:", str(datetime.datetime.now()-temp_time)
print "==================================="
print "Total Duration:", str(datetime.datetime.now()-start)

# IR SYSTEM READY
print "IR-SYSTEM READY ENTER QUERY AND PRESS ENTER:"
string = ""
while string != "quit":
    print "==============================================================="
    print "==============================================================="
    string = raw_input("Query: ").decode(sys.stdout.encoding)
    query = proc.process(string)
    result = sorted(idxr.search(query).iteritems(), key=operator.itemgetter(1),
                    reverse=False)
    for res in result:
        print str(res[1]) + " " + res[0].metadata["original"]
        print "---------------------------------------------------------------"
<commit_msg>using optparse to make the example more flexible<commit_after># -*- coding: utf-8 -*-
import datetime
import sys
import operator
import optparse
from pyretrieval import processor, document, indexer

if __name__ == "__main__":
    parser = optparse.OptionParser()
    parser.add_option("-s", "--stopwords", dest="stopwords")
    parser.add_option("-l", "--lemmas", dest="lemmas")
    parser.add_option("-d", "--documents", dest="documents")
    (options, args) = parser.parse_args()

    start = datetime.datetime.now()
    print "Initializing Document Processor"
    proc = processor.Processor()

    # LOAD STOPWORDS
    print "Loading Stopwords..."
    temp_time = datetime.datetime.now()
    proc.load_stopwords(options.stopwords, True)
    print "Finished after:", str(datetime.datetime.now()-temp_time)

    # LOAD LEMMAS
    print "Loading Lemmas..."
    temp_time = datetime.datetime.now()
    proc.load_lemmas(options.lemmas, True)
    print "Finished after:", str(datetime.datetime.now()-temp_time)

    # PROCESS DOCUMENTS
    print "Processing Documents"
    docs = []
    temp_time = datetime.datetime.now()
    with open(options.documents) as file:
        i = 0
        for line in file:
            i += 1
            sys.stdout.write(str(i)+'\r')
            docs.append(proc.process(line.decode("utf-8")))
    print "Finished after:", str(datetime.datetime.now()-temp_time)

    # INDEX DOCUMENTS
    print "Indexing Documents..."
    idxr = indexer.Indexer()
    temp_time = datetime.datetime.now()
    for doc in docs:
        idxr.index(doc)
    print "Finished after:", str(datetime.datetime.now()-temp_time)

    # CALCULATE INVERSE DOCUMENT FREQUENCY
    temp_time = datetime.datetime.now()
    print "Calculating inverse document frequency"
    idxr.calc_idfs()
    print "Finished after:", str(datetime.datetime.now()-temp_time)
    print "==================================="
    print "Total Duration:", str(datetime.datetime.now()-start)

    # IR SYSTEM READY
    print "IR-SYSTEM READY ENTER QUERY AND PRESS ENTER:"
    string = ""
    while string != "quit":
        print "==============================================================="
        print "==============================================================="
        string = raw_input("Query: ").decode(sys.stdout.encoding)
        query = proc.process(string)
        result = sorted(idxr.search(query).iteritems(), key=operator.itemgetter(1),
                        reverse=False)
        for res in result:
            print str(res[1]) + " " + res[0].metadata["original"]
            print "---------------------------------------------------------------"

{'fraction_non_alphanumeric': 0.21466942148760332, 'fraction_numerical': 0.002892561983471074, 'mean_word_length': 3.3889392565729826, 'pattern_counts': {'":': 3, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6829701', 'n_tokens_mistral': 1425, 'n_tokens_neox': 1369, 'n_words': 322}