<commit_before>''' 
Informations Retrieval Library
==============================
LM is an implementation of ngram Language Model 
'''

# Author: <NAME> <@gr33ndata> 

import sys, math

from superlist import SuperList
from progress import Progress
from preprocessor import Preprocessor

class UnseenTerms:

    def __init__(self):
        self.counts = {}
        
    def update(self, doc_id='', unseen=True):
        if not doc_id in self.counts:
            self.counts[doc_id] = {
                'seen': 0,
                'unseen': 0
            }       
        if unseen:
            self.counts[doc_id]['unseen'] += 1
        else:
            self.counts[doc_id]['seen'] += 1

    def display(self):
        for doc_id in self.counts:
            unseen = self.counts[doc_id]['unseen'] 
            total = self.counts[doc_id]['unseen'] + self.counts[doc_id]['seen']
            unseen_ratio =  unseen * 1.0 / total
            print 'Documdent:', doc_id
            print '\tUnseen ratio:', unseen_ratio 
    
    
class LM:

    
    def __init__(self, n=3, lpad='', rpad='', laplace_gama=1, 
                 verbose=False):
        '''
        Initialize our LM
        n: Pur LM's ngram, e.g. for bigram LM, n=2
        lpad, rpad: Left and right padding. 
                    If empty string '', then don't pad, else
                    For each document read pad terms
                    with n-1 repitition on lpad and/or rpad
        joiner: What to use to join ngrams, practically it doesn't matter
        '''
        self.n = n
        # Counters for joint probabilities
        # Count for w_1, w_2, w_3 ... w_{n-1}, w_n
        self.term_count_n = {}
        # Count for w_1, w_2, w_3 ... w_{n-1}
        self.term_count_n_1 = {}
        # The vocabulary of all classes (for Laplace smoothing)
        self.vocabulary = set()
        self.lpad=lpad
        self.rpad=rpad
        self.laplace_gama = laplace_gama
        self.joiner = ' '
        self.unseen_counts = UnseenTerms()
        self.verbose = verbose

    def display(self):
        '''
        Displays statistics about our LM
        '''
        self.self.unseen_counts.display()
    
    def to_ngrams(self, terms):
        ''' Converts terms to all possibe ngrams 
            terms: list of terms
        '''
        if len(terms) <= self.n:<commit_msg>refactoring LM and adding stats to it<commit_after>''' 
Informations Retrieval Library
==============================
LM is an implementation of ngram Language Model 
'''

# Author: <NAME> <@gr33ndata> 

import sys, math

from superlist import SuperList
from progress import Progress
from preprocessor import Preprocessor

class UnseenTerms:

    def __init__(self):
        self.counts = {}
        
    def update(self, doc_id='', unseen=True):
        if not doc_id in self.counts:
            self.counts[doc_id] = {
                'seen': 0,
                'unseen': 0
            }       
        if unseen:
            self.counts[doc_id]['unseen'] += 1
        else:
            self.counts[doc_id]['seen'] += 1

    def display(self):
        for doc_id in self.counts:
            unseen = self.counts[doc_id]['unseen'] 
            total = self.counts[doc_id]['unseen'] + self.counts[doc_id]['seen']
            unseen_ratio =  unseen * 1.0 / total
            print 'Documdent:', doc_id
            print '\tUnseen ratio:', unseen_ratio 
    
    
class LM:

    
    def __init__(self, n=3, lpad='', rpad='', laplace_gama=1, 
                 verbose=False):
        '''
        Initialize our LM
        n: Pur LM's ngram, e.g. for bigram LM, n=2
        lpad, rpad: Left and right padding. 
                    If empty string '', then don't pad, else
                    For each document read pad terms
                    with n-1 repitition on lpad and/or rpad
        joiner: What to use to join ngrams, practically it doesn't matter
        '''
        self.n = n
        # Counters for joint probabilities
        # Count for w_1, w_2, w_3 ... w_{n-1}, w_n
        self.term_count_n = {}
        # Count for w_1, w_2, w_3 ... w_{n-1}
        self.term_count_n_1 = {}
        # The vocabulary of all classes (for Laplace smoothing)
        self.vocabulary = set()
        self.lpad=lpad
        self.rpad=rpad
        self.laplace_gama = laplace_gama
        self.joiner = ' '
        self.unseen_counts = UnseenTerms()
        self.verbose = verbose

    def display(self):
        '''
        Displays statistics about our LM
        '''
        self.unseen_counts.display()
    
    def to_ngrams(self, terms):
        ''' Converts terms to all possibe ngrams 
            terms: list of terms
        '''
        if len(terms) <= self.n:
{'fraction_non_alphanumeric': 0.11403695052028032, 'fraction_numerical': 0.00891909110214483, 'mean_word_length': 1.78698224852071, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6361628', 'n_tokens_mistral': 1518, 'n_tokens_neox': 1422, 'n_words': 405}