<commit_before>from nltk import word_tokenize
import random
import linked_list


class NGramModel:
    def __init__(self, file, n):
        """
        :param file: path to text file to be used as training data
        :param n: the n for your n-gram model (2 uses bigrams etc)
        :return:
        """
        self.file = file
        self.n = n
        self.tokens = self.tokenise()  # linked list of tokens of text in order, including sentence start/end

    def tokenise(self):  # TODO test!
        """
        Tokenise the file using NLTK, add sentence start/end tokens.
        :return: linked list of tokens, including start/end of sentence tokens
        """
        # TODO exception handling!
        # read in file
        with open(self.file) as fin:
            text = fin.read()

            # tokenise the file using the NLTK word_tokenise() function
            tokens_list = word_tokenize(text)

        # we have a module object linked_list (imported) that provides a class called LinkedList()
        tokens_ll = linked_list.LinkedList()
        for tok in tokens_list:   # put these tokens into a linked list
            tokens_ll.list_insert_tail(tok)

        # insert sentence start token at start of text, sentence end token at end
        tokens_ll.list_insert_head("<s>")
        tokens_ll.list_insert_tail("</s>")

        # search tokens_ll for instances of _Node with key ".", when find one, insert two nodes: </s> and <s>
        x = tokens_ll.head  # Shouldn't it be _head? (I suppose we are using this from the module)  # no, cos @property
        while x is not None:
            if x.key == ".":  # when you find a full stop
                tokens_ll.list_insert_middle(x, "</s>")  # insert end of sentence marker
                tokens_ll.list_insert_middle(x.next_node, "<s>")  # and after that insert start of sentence marker
            x = x.next_node  # look at next node

        return tokens_ll

    def unigram_count(self):  # TODO test! compare to NLTK output?  this is now redundant...
        """
        Work out the counts of all the unigrams in the list of tokens.
        :return: dictionary of all unigrams and associated counts
        """
        unigram_dict = {}
        x = self.tokens.head
        while x is not None:
            if x.key not in unigram_dict:  # if you haven't encountered this token yet
                unigram_dict[x.key] = 1  # add a new entry to the dictionary, count 1
            else:  # you have already seen this token
                unigram_dict[x.key] += 1  # increment its count in the dictionary
            x = x.next_node
        return unigram_dict

    def ngram_count(self, n):  # TODO test. Compare with NLTK?
        """
        Work out the counts of all the n-grams in the list of tokens (generalised version of unigram count)
        :param n:
        :return:
        """
        ngram_dict = {}
        x = self.tokens.head  # start of n-gram
        y = x
        for i in range(n-1):
            y = y.next_node  # y represents the end of the n-gram
        while y is not None:  # ie while the n-gram won't go off the end of the list

            # build the key from the values of the nodes between x and y inclusive
            k = x.key  # the n-gram
            iter_node = x
            for i in range(n-1):
                iter_node = iter_node.next_node
                k = k + " " + iter_node.key

            # update the dictionary based on this key
            if k not in ngram_dict:
                ngram_dict[k] = 1
            else:
                ngram_dict[k] += 1

            # increment the pointers to the start and end of the n-gram
            x = x.next_node
            y = y.next_node
        return ngram_dict

    def make_bigram_model(self):
        word = self.tokens.head
        bigrams_lst = linked_list.LinkedList()  # this stores bigrams, key = first word, value = dict of possible second words and probabilities
        prev_word = ""
        while word:  # traverse the list of tokens to the end
            if prev_word != "":
                pass
                # found = bigrams_lst.find(prev_word.key)  # node if found, None if not
                # found_bigrams = bigrams_lst.find(prev_word.key)
                # if not found:  # prev_word doesn't have any bigrams yet
                    # add a new node to freq_lst with key prev_word.key and freq 1 and data {}
                    # add entry to this dictionary data[word] = 1
                # else:  # prev_word already has some bigrams
                    # found.freq += 1
                    # if word in found_bigrams.data:  # that bigram has been seen before
                        # found_bigrams.data[word] += 1  # increment its count
                    # else:   # not seen that bigram yet
                        # found_bigrams.data[word] = 1  # add a new entry to the dictionary
            prev_word = word
            word = word.next_node

        # turn counts into probabilities
        # firstword = bigrams_lst.head
        # while firstword:  # iterate through bigrams list
            # for secondword in firstword.data:
                # prob = firstword.data[secondword]/firstword.freq
                # firstword.data[secondword] = prob  # replace counts in bigrams dict with relative probability
        return bigrams_lst

    def generate_word(self, probDict):  # pass this a dictionary of probabilities (_Node.data)
        r = random.random()
        cumulative_prob = 0
        for word in probDict:
            cumulative_prob += probDict[word]
            if cumulative_prob > r:
                return word

    def generate_sentence(self):
        mod = self.make_bigram_model()
        text = []
        prev_word = "<s>"  # start with a beginning of sentence marker
        while prev_word != "<\s>":  # keep going until you find an end of sentence marker
            # node = mod.find(prev_word)
            # new_word = self.generate_word(node.data)
            # text.append(new_word)
            # prev_word = new_word
        return " ".join(text[1:-1])  # dont want to return the start/end of sentence markers

if __name__ == "__main__":
    mod = NGramModel("sml_test.txt", 2)
    # print(mod.tokenise())
    print(mod.ngram_count(3))<commit_msg>optimistic pseudocode in place for generating random word and sentence from hypothetical bigram model<commit_after>from nltk import word_tokenize
import random
import linked_list


class NGramModel:
    def __init__(self, file, n):
        """
        :param file: path to text file to be used as training data
        :param n: the n for your n-gram model (2 uses bigrams etc)
        :return:
        """
        self.file = file
        self.n = n
        self.tokens = self.tokenise()  # linked list of tokens of text in order, including sentence start/end

    def tokenise(self):  # TODO test!
        """
        Tokenise the file using NLTK, add sentence start/end tokens.
        :return: linked list of tokens, including start/end of sentence tokens
        """
        # TODO exception handling!
        # read in file
        with open(self.file) as fin:
            text = fin.read()

            # tokenise the file using the NLTK word_tokenise() function
            tokens_list = word_tokenize(text)

        # we have a module object linked_list (imported) that provides a class called LinkedList()
        tokens_ll = linked_list.LinkedList()
        for tok in tokens_list:   # put these tokens into a linked list
            tokens_ll.list_insert_tail(tok)

        # insert sentence start token at start of text, sentence end token at end
        tokens_ll.list_insert_head("<s>")
        tokens_ll.list_insert_tail("</s>")

        # search tokens_ll for instances of _Node with key ".", when find one, insert two nodes: </s> and <s>
        x = tokens_ll.head  # Shouldn't it be _head? (I suppose we are using this from the module)  # no, cos @property
        while x is not None:
            if x.key == ".":  # when you find a full stop
                tokens_ll.list_insert_middle(x, "</s>")  # insert end of sentence marker
                tokens_ll.list_insert_middle(x.next_node, "<s>")  # and after that insert start of sentence marker
            x = x.next_node  # look at next node

        return tokens_ll

    def unigram_count(self):  # TODO test! compare to NLTK output?  this is now redundant...
        """
        Work out the counts of all the unigrams in the list of tokens.
        :return: dictionary of all unigrams and associated counts
        """
        unigram_dict = {}
        x = self.tokens.head
        while x is not None:
            if x.key not in unigram_dict:  # if you haven't encountered this token yet
                unigram_dict[x.key] = 1  # add a new entry to the dictionary, count 1
            else:  # you have already seen this token
                unigram_dict[x.key] += 1  # increment its count in the dictionary
            x = x.next_node
        return unigram_dict

    def ngram_count(self, n):  # TODO test. Compare with NLTK?
        """
        Work out the counts of all the n-grams in the list of tokens (generalised version of unigram count)
        :param n:
        :return:
        """
        ngram_dict = {}
        x = self.tokens.head  # start of n-gram
        y = x
        for i in range(n-1):
            y = y.next_node  # y represents the end of the n-gram
        while y is not None:  # ie while the n-gram won't go off the end of the list

            # build the key from the values of the nodes between x and y inclusive
            k = x.key  # the n-gram
            iter_node = x
            for i in range(n-1):
                iter_node = iter_node.next_node
                k = k + " " + iter_node.key

            # update the dictionary based on this key
            if k not in ngram_dict:
                ngram_dict[k] = 1
            else:
                ngram_dict[k] += 1

            # increment the pointers to the start and end of the n-gram
            x = x.next_node
            y = y.next_node
        return ngram_dict

    def make_bigram_model(self):
        word = self.tokens.head
        bigrams_lst = linked_list.LinkedList()  # this stores bigrams, key = first word, value = dict of possible second words and probabilities
        prev_word = ""
        while word:  # traverse the list of tokens to the end
            if prev_word != "":
                pass
                # found = bigrams_lst.find(prev_word.key)  # node if found, None if not
                # found_bigrams = bigrams_lst.find(prev_word.key)
                # if not found:  # prev_word doesn't have any bigrams yet
                    # add a new node to freq_lst with key prev_word.key and freq 1 and data {}
                    # add entry to this dictionary data[word] = 1
                # else:  # prev_word already has some bigrams
                    # found.freq += 1
                    # if word in found_bigrams.data:  # that bigram has been seen before
                        # found_bigrams.data[word] += 1  # increment its count
                    # else:   # not seen that bigram yet
                        # found_bigrams.data[word] = 1  # add a new entry to the dictionary
            prev_word = word
            word = word.next_node

        # turn counts into probabilities
        # firstword = bigrams_lst.head
        # while firstword:  # iterate through bigrams list
            # for secondword in firstword.data:
                # prob = firstword.data[secondword]/firstword.freq
                # firstword.data[secondword] = prob  # replace counts in bigrams dict with relative probability
        return bigrams_lst

    def generate_word(self, probDict):  # pass this a dictionary of probabilities (_Node.data)
        r = random.random()
        cumulative_prob = 0
        for word in probDict:
            cumulative_prob += probDict[word]
            if cumulative_prob > r:
                return word

    def generate_sentence(self):
        mod = self.make_bigram_model()
        text = []
        prev_word = "<s>"  # start with a beginning of sentence marker
        while prev_word != "<\s>":  # keep going until you find an end of sentence marker
            pass
            # node = mod.find(prev_word)
            # new_word = self.generate_word(node.data)
            # text.append(new_word)
            # prev_word = new_word
        return " ".join(text[1:-1])  # dont want to return the start/end of sentence markers

if __name__ == "__main__":
    mod = NGramModel("sml_test.txt", 2)
    # print(mod.tokenise())
    print(mod.ngram_count(3))
{'fraction_non_alphanumeric': 0.07493112947658402, 'fraction_numerical': 0.002833530106257379, 'mean_word_length': 1.8456886898096305, 'pattern_counts': {'":': 8, '<': 19, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6223085', 'n_tokens_mistral': 3638, 'n_tokens_neox': 3470, 'n_words': 1319}