<commit_before>#-*- coding:utf8 -*-
import BaseHTTPServer
import sys
import socket
MACHINE_NETWORK_NAME = socket.gethostbyname(socket.gethostname())

port = int(sys.argv[1])
books_folder = sys.argv[2]
index_folder = sys.argv[3]
"""
port = 8334
books_folder = "/home/arslan/src/ngpedia/books1000"
index_folder ="indices/"
"""

from lib.search_engine import TSearchEngine
search_engine = TSearchEngine(index_location=index_folder)

for key, values in search_engine.word_index.keys_db.items():
    print key, values<commit_msg>warmup filling cache<commit_after>#-*- coding:utf8 -*-
import BaseHTTPServer
import sys
import socket
MACHINE_NETWORK_NAME = socket.gethostbyname(socket.gethostname())

port = int(sys.argv[1])
books_folder = sys.argv[2]
index_folder = sys.argv[3]
"""
port = 8334
books_folder = "/home/arslan/src/ngpedia/books1000"
index_folder ="indices/"
"""

from lib.search_engine import TSearchEngine
search_engine = TSearchEngine(index_location=index_folder)

top_freq = []
for key, values in search_engine.word_index.keys_db.items():
    top_freq += [(values[0], key)]
top_freq.sort(reverse=True)
top_freq = top_freq[:1005]
import urllib2
for _, token in top_freq:
    json_val = urllib2.urlopen("http://" + MACHINE_NETWORK_NAME + ":" + str(port) + "/?q=" + token + "&json=1").read()
    print json_val

{'fraction_non_alphanumeric': 0.13743356112376615, 'fraction_numerical': 0.024297646165527716, 'mean_word_length': 5.864583333333333, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6973705', 'n_tokens_mistral': 480, 'n_tokens_neox': 472, 'n_words': 92}