<commit_before>from flask import Flask, request, jsonify
import heapq

server = Flask(__name__, static_url_path='')

word_suggestions = {}
single_word_suggestions = []

@server.route('/')
def main():
    return server.send_static_file('index.html')

@server.route('/suggestion')
def suggestion():
    if 'q' not in request.args:
        return 500
    query = request.args['q']
    context = request.args.get('c', '')

    if context == '':
        suggestions = []
        for count, word in single_word_suggestions:
            if word.startswith(query):
                suggestions.append(word)
    else:
        count = len(word_suggestions['1.txt'][context])
        sorted_suggestions = heapq.nlargest(count, word_suggestions['1.txt'][context])
        suggestions = []
        for count, word in sorted_suggestions:
            if word.startswith(query):
                suggestions.append(word)

    return jsonify({'suggestions': suggestions})


def load_kb(name):
    word_dict = {}
    word_suggestions[name] = word_dict
    with open('kb/{}'.format(name), 'r') as f:
        current_gram = ''
        for line in f:
            if not line.startswith('\t'):
                current_gram = line.strip()
                word_dict[current_gram] = []
            else:
                word, count = line.strip().split(':')
                heapq.heappush(word_dict[current_gram], (int(count), word))

if __name__ == '__main__':
    # load autocomplete
    load_kb("0.txt")
    # for efficiency pre-sort and save all items since there is only one key
    count = len(word_suggestions['0.txt'][''])
    single_word_suggestions = heapq.nlargest(count, word_suggestions['0.txt'][''])

    load_kb("1.txt")

    server.run(host="0.0.0.0", port=8080)<commit_msg>allways initialize<commit_after>from flask import Flask, request, jsonify
import heapq

server = Flask(__name__, static_url_path='')

word_suggestions = {}
single_word_suggestions = []


def init():
    # load autocomplete
    load_kb("0.txt")
    # for efficiency pre-sort and save all items since there is only one key
    count = len(word_suggestions['0.txt'][''])
    single_word_suggestions = heapq.nlargest(count, word_suggestions['0.txt'][''])

    load_kb("1.txt")


init()

@server.route('/')
def main():
    return server.send_static_file('index.html')

@server.route('/suggestion')
def suggestion():
    if 'q' not in request.args:
        return 500
    query = request.args['q']
    context = request.args.get('c', '')

    if context == '':
        suggestions = []
        for count, word in single_word_suggestions:
            if word.startswith(query):
                suggestions.append(word)
    else:
        count = len(word_suggestions['1.txt'][context])
        sorted_suggestions = heapq.nlargest(count, word_suggestions['1.txt'][context])
        suggestions = []
        for count, word in sorted_suggestions:
            if word.startswith(query):
                suggestions.append(word)

    return jsonify({'suggestions': suggestions})


def load_kb(name):
    word_dict = {}
    word_suggestions[name] = word_dict
    with open('kb/{}'.format(name), 'r') as f:
        current_gram = ''
        for line in f:
            if not line.startswith('\t'):
                current_gram = line.strip()
                word_dict[current_gram] = []
            else:
                word, count = line.strip().split(':')
                heapq.heappush(word_dict[current_gram], (int(count), word))

if __name__ == '__main__':

    server.run(host="0.0.0.0", port=8080)
{'fraction_non_alphanumeric': 0.13871693866066404, 'fraction_numerical': 0.009566685424873381, 'mean_word_length': 2.7069864442127214, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5765864', 'n_tokens_mistral': 1146, 'n_tokens_neox': 1094, 'n_words': 235}