<commit_before># -*- coding: utf-8 -*-
# This file is part of open-tamil ngrams package
# (C) முத்தையா அண்ணாமலை 2013-2015,2017
# 
# N-gram language model for Tamil letters

import tamil
import copy
from .Corpus import Corpus
import codecs
import operator

class Letters:
    def __init__(self,filename):
        self.letter = dict()
        self.letter.update(zip( tamil.utf8.tamil_letters,
                                map(lambda x : 0, tamil.utf8.tamil_letters) ) )
        self.corpus = Corpus( filename )

    def __del__(self):
        try:
            del self.corpus
        except Exception:
            pass

    def __unicode__( self ):
        op = u""
        for lett,freq in self.letter.items():
            op = op + u"%s => %d\n"%(lett,freq)
        print(max(self.letter.values()))
        return op

    def save(self,filename):
        raise Exception('Not implemented')
    
class Unigram(Letters):
    def __init__(self,filename):
        Letters.__init__(self,filename)
        
    def frequency_model( self ):
        """ build a letter frequency model for Tamil letters from a corpus """
        # use a generator in corpus
        for next_letter in self.corpus.next_tamil_letter():
            # update frequency from corpus
            self.letter[next_letter] = self.letter[next_letter] + 1
    
    def save(self,filename):
        with codecs.open(filename,"w","utf-8") as fp:
            for k,v in sorted(self.letter.items(),key=operator.itemgetter(1),reverse=True):
                if v == 0:
                    continue
                fp.write(u"%s - %d\n"%(k,v))
        return True
    
class Bigram(Unigram):
    def __init__(self,filename):
        Unigram.__init__(self,filename)
        self.letter2 = dict()
        for k in tamil.utf8.tamil_letters:<commit_msg>iteratively process files to build N-gram corpus<commit_after># -*- coding: utf-8 -*-
# This file is part of open-tamil ngrams package
# (C) முத்தையா அண்ணாமலை 2013-2015,2017
# 
# N-gram language model for Tamil letters

import tamil
import copy
from .Corpus import Corpus
import codecs
import operator

class Letters:
    def __init__(self,filename):
        self.letter = dict()
        self.letter.update(zip( tamil.utf8.tamil_letters,
                                map(lambda x : 0, tamil.utf8.tamil_letters) ) )
        self.corpus = Corpus( filename )

    def __del__(self):
        try:
            del self.corpus
        except Exception:
            pass

    def __unicode__( self ):
        op = u""
        for lett,freq in self.letter.items():
            op = op + u"%s => %d\n"%(lett,freq)
        print(max(self.letter.values()))
        return op
    
    def update_file(self,filename):
        self.corpus = Corpus( filename )
        
    def save(self,filename):
        raise Exception('Not implemented')
    
class Unigram(Letters):
    def __init__(self,filename):
        Letters.__init__(self,filename)
        
    def frequency_model( self ):
        """ build a letter frequency model for Tamil letters from a corpus """
        # use a generator in corpus
        for next_letter in self.corpus.next_tamil_letter():
            # update frequency from corpus
            self.letter[next_letter] = self.letter[next_letter] + 1
    
    def save(self,filename):
        with codecs.open(filename,"w","utf-8") as fp:
            for k,v in sorted(self.letter.items(),key=operator.itemgetter(1),reverse=True):
                if v == 0:
                    continue
                fp.write(u"%s - %d\n"%(k,v))
        return True
    
class Bigram(Unigram):
    def __init__(self,filename):
        Unigram.__init__(self,filename)
        self.letter2 = dict()
        for k in tamil.utf8.tamil_letters:
{'fraction_non_alphanumeric': 0.11140796152818595, 'fraction_numerical': 0.011755276516163505, 'mean_word_length': 2.1330543933054393, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4700380', 'n_tokens_mistral': 1191, 'n_tokens_neox': 1146, 'n_words': 292}