<commit_before>from django.shortcuts import render
from django.http import HttpResponse
from json import dumps
from engine.query import QueryProcessor
from engine.invdx import InvertedIndex, DocumentLengthTable, WordFrequencyTable
from engine.parse import CorpusParser
import os

# Create your views here.

# set up data structures
dlt = DocumentLengthTable()
dlt.read('index/length.txt')
wft = WordFrequencyTable()
wft.read('index/freq.txt')
proc = QueryProcessor('index/index.txt', dlt, wft)

# read documents for retrieval
cp = CorpusParser('index/corpus.txt')
cp.parse()
docs = cp.get_corpus()


def index(request):
	return render(request, 'search/index.html', {})


def query(request):
	query = request.GET.get('query').split(' ')
	results = proc.run_query_likelihood(query)
	return HttpResponse(dumps(results))


def doc(request):
	docid = str(request.GET.get('doc'))
	text = {'text': docs[docid]}
	return HttpResponse(dumps(text))<commit_msg>Get document text from database<commit_after>from django.shortcuts import render
from django.http import HttpResponse
from json import dumps
from engine.query import QueryProcessor
from engine.invdx import InvertedIndex, DocumentLengthTable, WordFrequencyTable
from engine.parse import CorpusParser
from engine.db import cursor
import os

# Create your views here.

# set up data structures
dlt = DocumentLengthTable()
dlt.read('index/length.txt')
wft = WordFrequencyTable()
wft.read('index/freq.txt')
proc = QueryProcessor('index/index.txt', dlt, wft)

# read documents for retrieval
cp = CorpusParser('index/corpus.txt')
cp.parse()
docs = cp.get_corpus()


def index(request):
	return render(request, 'search/index.html', {})


def query(request):
	query = request.GET.get('query').split(' ')
	results = proc.run_query_likelihood(query)
	return HttpResponse(dumps(results))


def doc(request):
	docid = int(request.GET.get('doc'))
	curs = cursor()
	curs.execute('SELECT text FROM Document WHERE docid="%d"' % docid)
	doc_text, = curs.fetchone()
	text = {'text': doc_text}
	return HttpResponse(dumps(text))


if __name__ == '__main__':
	cp = CorpusParser('index/corpus.txt')
	cp.parse()
	docs = cp.get_corpus()
{'fraction_non_alphanumeric': 0.125, 'fraction_numerical': 0.0, 'mean_word_length': 5.904153354632588, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5062987', 'n_tokens_mistral': 694, 'n_tokens_neox': 677, 'n_words': 180}