<commit_before>from flask import Flask
from os import environ, mkdir
from os.path import exists,join
from shutil import rmtree
import os.path
#import whoosh
#import whoosh.fields, whoosh.index
import pymongo
from pymongo import MongoClient
import sys

#whoosh_dir = join(environ["OPENSHIFT_DATA_DIR"],'index')
#
#if True or not exists(whoosh_dir):
#    rmtree(whoosh_dir)
#    mkdir(whoosh_dir)
#
#    schema = whoosh.fields.Schema(
#        title = whoosh.fields.TEXT(),
#        authors = whoosh.fields.TEXT(),
#        description = whoosh.fields.TEXT()
#    )
#    ix = whoosh.index.create_in(whoosh_dir, schema)
#else:
#    ix = whoosh.index.open_ix(whoosh_dir)

app = Flask(__name__)

mongoclient = pymongo.MongoClient(environ["OPENSHIFT_MONGODB_DB_URL"])
db = mongoclient.database

@app.route('/eimer/add/<thing>')
def add_to_collection(thing):
    db.eimer.insert({"thing" : thing})
    return "%s added" % thing

@app.route('/eimer/view')
def view_collection():
    return " ".join([doc["thing"] for doc in db.eimer.find()])


if __name__ == '__main__':
    app.run()<commit_msg>searching<commit_after>from flask import Flask
from os import environ, mkdir
from os.path import exists,join
from shutil import rmtree
import os.path
import whoosh
import whoosh.fields, whoosh.index
from whoosh.query import *
import pymongo
from pymongo import MongoClient
import sys

whoosh_dir = join(environ["OPENSHIFT_DATA_DIR"],'index')

if True or not exists(whoosh_dir):
    rmtree(whoosh_dir)
    mkdir(whoosh_dir)

    schema = whoosh.fields.Schema(
        thing = whoosh.fields.TEXT(),
        subthing = whoosh.fields.TEXT()
    )
    ix = whoosh.index.create_in(whoosh_dir, schema)
else:
    ix = whoosh.index.open_ix(whoosh_dir)

app = Flask(__name__)

mongoclient = pymongo.MongoClient(environ["OPENSHIFT_MONGODB_DB_URL"])
db = mongoclient.database
db.eimer.drop()

@app.route('/eimer/add/<thing>/<subthing>')
def add_to_collection(thing,subthing):
    db.eimer.insert({"thing" : thing, "subthing" : subthing})

    #TODO: handle non-thread safety
    writer = ix.writer()
    writer.add_document(thing=thing, subthing=subthing)
    writer.commit()
    return "%s added" % thing

@app.route('/eimer/view')
def view_collection():
    return " ".join([doc["thing"] for doc in db.eimer.find()])

@app.route('/eimer/search/<term>')
def search_collection(term):
    with ix.searcher() as searcher:
        query = Or([Term("thing",term), Term("subthing",term)])
        results = searcher.search(query)
        if len(results) == 0:
            return "No results found"
        else:
            " ".join(res["thing"] for res in results)


if __name__ == '__main__':
    app.run()
{'fraction_non_alphanumeric': 0.1362448674878686, 'fraction_numerical': 0.0003732736095558044, 'mean_word_length': 4.056603773584905, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1404349', 'n_tokens_mistral': 946, 'n_tokens_neox': 931, 'n_words': 203}