<commit_before>__author__ = 'rose'

class LanguageModel():
    """
    manages reading in a file and splitting into a term, occurrence dictionary
    or takes a term, occurrence dictionary
    calculates the probability of a term occurring given the occurrence dict
    """

    #TODO(leifos): dict is a reserved word - i.e. defines a dict.. be mindful of using dict, list, etc unless you are creating one.
    def __init__(self, file=None , occurrences_dict=None):
        self.occurrence_dict = {}
        self.TOTAL_OCCURRENCES = 0

        if file:
            self._populate_occurrences(file)
        else:
            self.occurrence_dict = occurrences_dict

        self._calc_total_occurrences()


    def _populate_occurrences(self, file_name):
        """
        reads in a file and stores in occurrences dictionary
        :param fileName
        """
        if file_name:
            f = open(file_name, 'r')
            for line in f:
                split_line=line.split()
                term = split_line[0]
                #TODO need to make robust for errors in input file
                count = int(split_line[1])
                self.occurrence_dict[term] = count

    def _calc_total_occurrences(self):
        """
        counts the total number of term occurrences in occurrence_dict
        """
        for term, count in self.occurrence_dict.items():
            self.TOTAL_OCCURRENCES += count

    def get_total_occurrences(self):
        return self.TOTAL_OCCURRENCES

    #TODO(leifos): be consisent, this was get_number_terms.. it was used as get_num_terms
    def get_num_terms(self):
        return len(self.occurrence_dict)

    def get_num_occurrences(self,term):
        """
        :param term: the individual term for which count is desired
        :return: number of occurrences for term in occurrence dict, 0 if not in
        """
        if term in self.occurrence_dict:
            return self.occurrence_dict[term]
        else:
            return 0

    def get_term_prob(self, term):
        """
        calculates the probability of a term
        :param term:
        :return:
        """
        if self.get_num_occurrences(term) == 0:
            return 0
        else:
            result =float(self.get_num_occurrences(term))/float(self.get_total_occurrences())
            result = round(result,2)
            return result

<commit_msg>refactoring<commit_after>__author__ = 'rose'

class LanguageModel():
    """
    manages reading in a file and splitting into a term, occurrence dictionary
    or takes a term, occurrence dictionary
    calculates the probability of a term occurring given the occurrence dict
    """

    #TODO(leifos): dict is a reserved word - i.e. defines a dict.. be mindful of using dict, list, etc unless you are creating one.
    def __init__(self, file=None , occurrences_dict=None):
        self.occurrence_dict = {}
        self.TOTAL_OCCURRENCES = 0

        if file:
            self._populate_occurrences(file)
        else:
            self.occurrence_dict = occurrences_dict

        self._calc_total_occurrences()


    def _populate_occurrences(self, file_name):
        """
        reads in a file and stores in occurrences dictionary
        :param fileName
        """
        if file_name:
            f = open(file_name, 'r')
            for line in f:
                split_line=line.split()
                term = split_line[0]
                #TODO need to make robust for errors in input file
                count = int(split_line[1])
                self.occurrence_dict[term] = count

    def _calc_total_occurrences(self):
        """
        counts the total number of term occurrences in occurrence_dict
        """
        for term, count in self.occurrence_dict.items():
            self.TOTAL_OCCURRENCES += count

    def get_total_occurrences(self):
        return self.TOTAL_OCCURRENCES

    #TODO(leifos): be consisent, this was get_number_terms.. it was used as get_num_terms
    def get_num_terms(self):
        return len(self.occurrence_dict)

    def get_num_occurrences(self,term):
        """
        :param term: the individual term for which count is desired
        :return: number of occurrences for term in occurrence dict, 0 if not in
        """
        if term in self.occurrence_dict[term]:
            return self.occurrence_dict[term]
        else:
            return 0

    def get_term_prob(self, term):
        """
        calculates the probability of a term
        :param term:
        :return:
        """
        if self.get_num_occurrences(term) == 0:
            return 0
        else: 
            result =float(self.get_num_occurrences(term))/float(self.get_total_occurrences())
            result = round(result,2)
            return result


{'fraction_non_alphanumeric': 0.07603927303112597, 'fraction_numerical': 0.0033423856277418006, 'mean_word_length': 2.121251629726206, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '219747', 'n_tokens_mistral': 1446, 'n_tokens_neox': 1315, 'n_words': 408}