<commit_before>import marisa_trie as marisa
import pandas as pd
import numpy as np
import codecs
import pickle
import sys

# TODO: Need to rewrite the file read. Right now it assumes that only one text file is used.
# The easiest way is to probably change load_data to combine 2 and 3-grams(and maybe more)

def autocomplete(start_word, data_path, triee=None):
    """
    Autocomplete a word by finding the most widely used n-gram starting with it.
    :param start_word: a word to autocomplete
    :param data_path: path to the word corpus
    :param triee: an optional argument in special case where a trie already exists, main purpose
    is debugging
    :return: a string with the autocompleted word
    """

    highest_freq = 0
    complete_word = ''

    # Get an appropriate trie
    if triee is None:
        triee, popular_dict = check_cache(data_path, start_word)


    # Iterate over the trie elements that start with the start_word
    # and store the top 3 most frequent words
    item = triee.items(start_word)

    # complete_word = []
    # complete_freq = []
    # i = 0

    if len(start_word.split(" ")) < 1:
        next_word = next_word_indicator(item, start_word)
    else:
        next_word = False
        item = list(map(lambda x: (x[0].replace(start_word.split(" ")[0] + " ", ""), x[1]), item))
    # while len(item) > 0 and i < 3:
    #     highest_freq = 0
    #     for key, val in item:
    #         if val[0] > highest_freq:
    #             highest_freq = val[0]
    #             if key not in complete_word:
    #                 complete_word.append(key)
    #                 complete_freq.append(val)
    #     if not next_word:
    #         item = [(k, v) for k, v in item if complete_word[0].split(" ")[0] not in k]
    #     if complete_word[-1] in item:
    #         item.remove((complete_word[-1],complete_freq[-1]))
    #     i += 1


    # complete_freq_vals = list(map(lambda x: x[0], complete_freq))
    # complete_word_dict = dict(zip(complete_word, complete_freq_vals))
    complete_word_dict = dict(item)
    top_three = []
    while len(top_three) < 3:
        if len(complete_word_dict) > 0:
            pop_word = max(complete_word_dict, key=complete_word_dict.get)
            complete_word_dict.pop(pop_word, None)
            print(pop_word)
        else:
            pop_word = max(popular_dict, key=popular_dict.get)
            popular_dict.pop(pop_word, None)

        if pop_word not in top_three:
            if not next_word:
                if pop_word.split(" ")[0] not in top_three:
                    top_three.append(pop_word.split(" ")[0])
            else:
                top_three.append(pop_word)

    complete_word = top_three
<commit_msg>small changes<commit_after>import marisa_trie as marisa
import pandas as pd
import numpy as np
import codecs
import pickle
import sys

# TODO: Need to rewrite the file read. Right now it assumes that only one text file is used.
# The easiest way is to probably change load_data to combine 2 and 3-grams(and maybe more)

def autocomplete(start_word, data_path, triee=None):
    """
    Autocomplete a word by finding the most widely used n-gram starting with it.
    :param start_word: a word to autocomplete
    :param data_path: path to the word corpus
    :param triee: an optional argument in special case where a trie already exists, main purpose
    is debugging
    :return: a string with the autocompleted word
    """

    highest_freq = 0
    complete_word = ''

    # Get an appropriate trie
    if triee is None:
        triee, popular_dict = check_cache(data_path, start_word)


    # Iterate over the trie elements that start with the start_word
    # and store the top 3 most frequent words
    item = triee.items(start_word)

    # complete_word = []
    # complete_freq = []
    # i = 0

    if len(start_word.split(" ")) < 1:
        next_word = next_word_indicator(item, start_word)
    else:
        next_word = False
        item = list(map(lambda x: (x[0].replace(start_word.split(" ")[0] + " ", ""), x[1]), item))
    # while len(item) > 0 and i < 3:
    #     highest_freq = 0
    #     for key, val in item:
    #         if val[0] > highest_freq:
    #             highest_freq = val[0]
    #             if key not in complete_word:
    #                 complete_word.append(key)
    #                 complete_freq.append(val)
    #     if not next_word:
    #         item = [(k, v) for k, v in item if complete_word[0].split(" ")[0] not in k]
    #     if complete_word[-1] in item:
    #         item.remove((complete_word[-1],complete_freq[-1]))
    #     i += 1


    # complete_freq_vals = list(map(lambda x: x[0], complete_freq))
    # complete_word_dict = dict(zip(complete_word, complete_freq_vals))
    complete_word_dict = dict(item)
    top_three = []
    while len(top_three) < 3:
        if len(complete_word_dict) > 0:
            pop_word = max(complete_word_dict, key=complete_word_dict.get)
            complete_word_dict.pop(pop_word, None)
            # print(pop_word)
        else:
            pop_word = max(popular_dict, key=popular_dict.get)
            popular_dict.pop(pop_word, None)

        if pop_word not in top_three:
            if not next_word:
                if pop_word.split(" ")[0] not in top_three:
                    top_three.append(pop_word.split(" ")[0])
            else:
                top_three.append(pop_word)

    complete_word = top_three

{'fraction_non_alphanumeric': 0.09389930172730614, 'fraction_numerical': 0.009187798603454611, 'mean_word_length': 2.3828464885021754, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1651290', 'n_tokens_mistral': 1749, 'n_tokens_neox': 1676, 'n_words': 517}