<commit_before>#! /usr/bin/env/python2
"""
    Tree Based Statistical language Model
    to be used in tandem with an acoustic model
    for speech recognition
"""


from collections import Counter   # todo : remove this
from node import Node
from question import Question
import logging
import math
import preprocess   # todo check for namespace pollution
import Queue


class Trsl(object):
    """
        Trsl class implements a tree based statistical language model
        Arguments:
            self.reduction_threshold -> min reduction in entropy for a question
                                        at a node to further grow the tree
            self.ngram_window_size   -> no of predictor variables (inclusive of target)
    """

    def __init__(self, ngram_window_size=5, reduction_threshold=0.15):

        self.reduction_threshold = reduction_threshold
        self.ngram_window_size = ngram_window_size
        self.root = Node()
        self.previous_best_questions = set()
        self.no_of_nodes = 1
        self.ngram_table = None
        self.vocabulary_set = None
        self.word_sets = None
        # inf queue size hopefully <- todo : evaluate if we need this
        self.node_queue = Queue.Queue(-1)

    def train(self, filename):
        """
            Given a filename(containing a training corpus), build a decision tree
            which can be accessed through self.root

        """

        self.ngram_table, self.vocabulary_set = preprocess.preprocess(
            filename, self.ngram_window_size
        )
        self.set_root_state()
        self.node_queue.put(self.root)
        self.word_sets = self.build_sets()
        while not self.node_queue.empty():
            self.process_node(self.node_queue.get())
        logging.info("Total no of Nodes:"+ str(self.no_of_nodes))

    def set_root_state(self):
        """
            Calculates the probability distribution
            of the target variable values at the root node
            in the training text and also
            the entropy of this distribution.
            Also, adds all the row indices in the ngram table
            into row_fragment_indices since the data is not
            fragmented yet.
        """
        self.root.dist = {}
        for index in range(0, len(self.ngram_table)):
            try:
                self.root.dist[
                    self.ngram_table[index, self.ngram_window_size-1]
                    ] += 1.0
            except KeyError:
                self.root.dist[
                    self.ngram_table[index, self.ngram_window_size-1]
                    ] = 1.0

        # todo check if computes through entrie ds or not
        self.root.row_fragment_indices = [
            x for x in range(0, len(self.ngram_table))
        ]

        for key in self.root.dist.keys():
            frequency = self.root.dist[key]
            probability = frequency/len(self.root.row_fragment_indices)
            probability_of_info_gain = probability * math.log(probability, 2)
            self.root.dist[key] = probability
            self.root.entropy += -probability_of_info_gain

    def process_node(self, curr_node):
        """
            Used to process curr_node by computing best reduction
            and choosing to create children nodes or not based on
            self.reduction_threshold

            Naming Convention:
                *    nb  -> Not belongs the the selected set
                *    b   -> Belongs to the selected set
                *    Xi  -> Predictor variable index into the ngram table
                *    Si  -> Set
                *    cnb -> current node not belongs to the selected set
                *    cb  -> current node belongs to the set
        """

        global logging

        best_question = Question()
        self.no_of_nodes += 1
        for Xi in range(0, self.ngram_window_size-1):
            for Si in self.word_sets:
                if self.question_already_asked(curr_node, Xi, Si):
                    continue
                curr_question = Question()
                for table_index in curr_node.row_fragment_indices:
                    predictor_word = self.ngram_table[table_index, Xi]
                    target_word = self.ngram_table[
                        table_index, self.ngram_window_size-1
                    ]
                    if predictor_word in Si:
                        curr_question.b_indices.append(table_index)
                        try:
                            curr_question.b_dist[target_word] += 1.0
                        except KeyError:
                            curr_question.b_dist[target_word] = 1.0
                    else:
                        curr_question.nb_indices.append(table_index)
                        try:
                            curr_question.nb_dist[target_word] += 1.0
                        except KeyError:
                            curr_question.nb_dist[target_word] = 1.0
                b_frequency_sum = sum(curr_question.b_dist.values())
                for key in curr_question.b_dist.keys():
                    frequency = curr_question.b_dist[key]
                    probability = frequency/b_frequency_sum
                    probability_of_info_gain = (
                        probability * math.log(probability, 2)
                    )
                    curr_question.b_dist[key] = probability
                    curr_question.b_dist_entropy += -probability_of_info_gain

                nb_frequency_sum = sum(curr_question.nb_dist.values())
                for key in curr_question.nb_dist.keys():
                    frequency = curr_question.nb_dist[key]
                    probability = frequency/nb_frequency_sum
                    probability_of_info_gain = (
                        probability * math.log(probability, 2)
                    )
                    curr_question.nb_dist[key] = probability
                    curr_question.nb_dist_entropy += -probability_of_info_gain
                size_row_fragment = (
                    len(curr_node.row_fragment_indices)
                )
                curr_question.b_probability = (
                    float(len(curr_question.b_indices))/size_row_fragment
                )
                curr_question.nb_probability = (
                    float(len(curr_question.nb_indices))/size_row_fragment
                )
                curr_question.avg_conditional_entropy = (
                    (curr_question.b_probability
                        * curr_question.b_dist_entropy)
                    +
                    (curr_question.nb_probability
                        * curr_question.nb_dist_entropy)
                )
                curr_question.reduction = (
                    curr_node.entropy - curr_question.avg_conditional_entropy
                )
                logging.debug(
                    "Predictor var: " + str(Xi)
                    + " Set: " + str(Si)
                    + " Reduction: " + str(curr_question.reduction)
                )
                if best_question.reduction < curr_question.reduction:
                    best_question = curr_question

        if best_question.reduction > self.reduction_threshold:
            curr_node.set = best_question.set
            curr_node.predictor_variable_index = (
                best_question.predictor_variable_index
            )
            curr_node.lchild = Node()
            curr_node.lchild.row_fragment_indices = best_question.b_indices
            curr_node.lchild.entropy = best_question.b_dist_entropy
            curr_node.lchild.dist = best_question.b_dist
            curr_node.rchild = Node()
            curr_node.rchild.row_fragment_indices = best_question.nb_indices
            curr_node.rchild.entropy = best_question.nb_dist_entropy
            curr_node.rchild.dist = best_question.nb_dist<commit_msg>Bugfix -> set and predictive variables was not being set on current question<commit_after>#! /usr/bin/env/python2
"""
    Tree Based Statistical language Model
    to be used in tandem with an acoustic model
    for speech recognition
"""


from collections import Counter   # todo : remove this
from node import Node
from question import Question
import logging
import math
import preprocess   # todo check for namespace pollution
import Queue


class Trsl(object):
    """
        Trsl class implements a tree based statistical language model
        Arguments:
            self.reduction_threshold -> min reduction in entropy for a question
                                        at a node to further grow the tree
            self.ngram_window_size   -> no of predictor variables (inclusive of target)
    """

    def __init__(self, ngram_window_size=5, reduction_threshold=0.15):

        self.reduction_threshold = reduction_threshold
        self.ngram_window_size = ngram_window_size
        self.root = Node()
        self.previous_best_questions = set()
        self.no_of_nodes = 1
        self.ngram_table = None
        self.vocabulary_set = None
        self.word_sets = None
        # inf queue size hopefully <- todo : evaluate if we need this
        self.node_queue = Queue.Queue(-1)

    def train(self, filename):
        """
            Given a filename(containing a training corpus), build a decision tree
            which can be accessed through self.root

        """

        self.ngram_table, self.vocabulary_set = preprocess.preprocess(
            filename, self.ngram_window_size
        )
        self.set_root_state()
        self.node_queue.put(self.root)
        self.word_sets = self.build_sets()
        while not self.node_queue.empty():
            self.process_node(self.node_queue.get())
        logging.info("Total no of Nodes:"+ str(self.no_of_nodes))

    def set_root_state(self):
        """
            Calculates the probability distribution
            of the target variable values at the root node
            in the training text and also
            the entropy of this distribution.
            Also, adds all the row indices in the ngram table
            into row_fragment_indices since the data is not
            fragmented yet.
        """
        self.root.dist = {}
        for index in range(0, len(self.ngram_table)):
            try:
                self.root.dist[
                    self.ngram_table[index, self.ngram_window_size-1]
                    ] += 1.0
            except KeyError:
                self.root.dist[
                    self.ngram_table[index, self.ngram_window_size-1]
                    ] = 1.0

        # todo check if computes through entrie ds or not
        self.root.row_fragment_indices = [
            x for x in range(0, len(self.ngram_table))
        ]

        for key in self.root.dist.keys():
            frequency = self.root.dist[key]
            probability = frequency/len(self.root.row_fragment_indices)
            probability_of_info_gain = probability * math.log(probability, 2)
            self.root.dist[key] = probability
            self.root.entropy += -probability_of_info_gain

    def process_node(self, curr_node):
        """
            Used to process curr_node by computing best reduction
            and choosing to create children nodes or not based on
            self.reduction_threshold

            Naming Convention:
                *    nb  -> Not belongs the the selected set
                *    b   -> Belongs to the selected set
                *    Xi  -> Predictor variable index into the ngram table
                *    Si  -> Set
                *    cnb -> current node not belongs to the selected set
                *    cb  -> current node belongs to the set
        """

        global logging

        best_question = Question()
        self.no_of_nodes += 1
        for Xi in range(0, self.ngram_window_size-1):
            for Si in self.word_sets:
                if self.question_already_asked(curr_node, Xi, Si):
                    continue
                curr_question = Question()
                curr_question.set = Si
                curr_question.predictor_variable_index = Xi
                for table_index in curr_node.row_fragment_indices:
                    predictor_word = self.ngram_table[table_index, Xi]
                    target_word = self.ngram_table[
                        table_index, self.ngram_window_size-1
                    ]
                    if predictor_word in Si:
                        curr_question.b_indices.append(table_index)
                        try:
                            curr_question.b_dist[target_word] += 1.0
                        except KeyError:
                            curr_question.b_dist[target_word] = 1.0
                    else:
                        curr_question.nb_indices.append(table_index)
                        try:
                            curr_question.nb_dist[target_word] += 1.0
                        except KeyError:
                            curr_question.nb_dist[target_word] = 1.0
                b_frequency_sum = sum(curr_question.b_dist.values())
                for key in curr_question.b_dist.keys():
                    frequency = curr_question.b_dist[key]
                    probability = frequency/b_frequency_sum
                    probability_of_info_gain = (
                        probability * math.log(probability, 2)
                    )
                    curr_question.b_dist[key] = probability
                    curr_question.b_dist_entropy += -probability_of_info_gain

                nb_frequency_sum = sum(curr_question.nb_dist.values())
                for key in curr_question.nb_dist.keys():
                    frequency = curr_question.nb_dist[key]
                    probability = frequency/nb_frequency_sum
                    probability_of_info_gain = (
                        probability * math.log(probability, 2)
                    )
                    curr_question.nb_dist[key] = probability
                    curr_question.nb_dist_entropy += -probability_of_info_gain
                size_row_fragment = (
                    len(curr_node.row_fragment_indices)
                )
                curr_question.b_probability = (
                    float(len(curr_question.b_indices))/size_row_fragment
                )
                curr_question.nb_probability = (
                    float(len(curr_question.nb_indices))/size_row_fragment
                )
                curr_question.avg_conditional_entropy = (
                    (curr_question.b_probability
                        * curr_question.b_dist_entropy)
                    +
                    (curr_question.nb_probability
                        * curr_question.nb_dist_entropy)
                )
                curr_question.reduction = (
                    curr_node.entropy - curr_question.avg_conditional_entropy
                )
                logging.debug(
                    "Predictor var: " + str(Xi)
                    + " Set: " + str(Si)
                    + " Reduction: " + str(curr_question.reduction)
                )
                if best_question.reduction < curr_question.reduction:
                    best_question = curr_question
                    logging.debug("Best Question : "+str(best_question.set))

        if best_question.reduction > self.reduction_threshold:
            curr_node.set = best_question.set
            curr_node.predictor_variable_index = (
                best_question.predictor_variable_index
            )
            curr_node.lchild = Node()
            curr_node.lchild.row_fragment_indices = best_question.b_indices
            curr_node.lchild.entropy = best_question.b_dist_entropy
            curr_node.lchild.dist = best_question.b_dist
            curr_node.rchild = Node()
            curr_node.rchild.row_fragment_indices = best_question.nb_indices
            curr_node.rchild.entropy = best_question.nb_dist_entropy
            curr_node.rchild.dist = best_question.nb_dist
{'fraction_non_alphanumeric': 0.06589703588143525, 'fraction_numerical': 0.00374414976599064, 'mean_word_length': 1.5494750238625516, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4747496', 'n_tokens_mistral': 4259, 'n_tokens_neox': 4016, 'n_words': 915}