<commit_before>import sys
import codecs

#from dyslib.lm import LM
from social import SocialLM as LM
from corpora.corpuslib import Train

class LangID(LM):

    def __init__(self, unk=False):

        # Shall we mark some text as unk,
        # if top languages are too close?
        self.unk = unk
        self.min_karbasa = 0.08

        # LM Parameters
        ngram = 3
        lrpad = u' '
        verbose=False
        corpus_mix='l'

        self.lm = LM(n=ngram, verbose=verbose, lpad=lrpad, rpad=lrpad, 
                smoothing='Laplace', laplace_gama=0.1, 
                corpus_mix=corpus_mix)

    def _readfile(self, filename):

        f = codecs.open(filename, encoding='utf-8')
        filedata = f.read()
        f.close()
        tokenz = LM.tokenize(filedata, mode='c')
        #print tokenz
        return tokenz

    def train(self, root=''):

        self.train = Train(root=root)
        corpus = self.train.get_corpus()

        # Show loaded Languages
        #print 'Lang Set: ' + ' '.join(train.get_lang_set())

        for item in corpus:
            self.lm.add_doc(doc_id=item[0], doc_terms=self._readfile(item[1]))

    
    def add_training_sample(self, text=u'', lang=''):
        
        self.train.add(text=text, lang=lang)

    def save_training_samples(self, domain='', filename=''):

        self.train.save(domain=domain, filename=filename)

    def classify(self, text=u''):

        text = self.lm.normalize(text)
        tokenz = LM.tokenize(text, mode='c')
        result = self.lm.calculate(doc_terms=tokenz)
        #print 'Karbasa:', self.karbasa(result)
        if self.unk and self.karbasa(result) < self.min_karbasa:
            lang = 'unk'
        else:
            lang = result['calc_id']
        return lang

if __name__ == '__main__':


    l = LangID(unk=False)
    #l.train()

    l.train('/Users/tarekamr/Desktop/dysly')

    l.add_training_sample(text=u'ellohay orldway', lang='pig')
    #l.save_training_samples()
    l.save_training_samples(domain='domaingo', filename='zofile.txt')
    sys.exit('Done!')

    if len(sys.argv) > 1:
        text = u' '.join(sys.argv[1:])
    else:
        text = u'hello, world'

    lang = l.classify(text)
    print text, '[ Language:', lang, ']'
<commit_msg>Removing unneeded example<commit_after>import sys
import codecs

#from dyslib.lm import LM
from social import SocialLM as LM
from corpora.corpuslib import Train

class LangID(LM):

    def __init__(self, unk=False):

        # Shall we mark some text as unk,
        # if top languages are too close?
        self.unk = unk
        self.min_karbasa = 0.08

        # LM Parameters
        ngram = 3
        lrpad = u' '
        verbose=False
        corpus_mix='l'

        self.lm = LM(n=ngram, verbose=verbose, lpad=lrpad, rpad=lrpad, 
                smoothing='Laplace', laplace_gama=0.1, 
                corpus_mix=corpus_mix)

    def _readfile(self, filename):

        f = codecs.open(filename, encoding='utf-8')
        filedata = f.read()
        f.close()
        tokenz = LM.tokenize(filedata, mode='c')
        #print tokenz
        return tokenz

    def train(self, root=''):

        self.train = Train(root=root)
        corpus = self.train.get_corpus()

        # Show loaded Languages
        #print 'Lang Set: ' + ' '.join(train.get_lang_set())

        for item in corpus:
            self.lm.add_doc(doc_id=item[0], doc_terms=self._readfile(item[1]))

    
    def add_training_sample(self, text=u'', lang=''):
        
        self.train.add(text=text, lang=lang)

    def save_training_samples(self, domain='', filename=''):

        self.train.save(domain=domain, filename=filename)

    def classify(self, text=u''):

        text = self.lm.normalize(text)
        tokenz = LM.tokenize(text, mode='c')
        result = self.lm.calculate(doc_terms=tokenz)
        #print 'Karbasa:', self.karbasa(result)
        if self.unk and self.karbasa(result) < self.min_karbasa:
            lang = 'unk'
        else:
            lang = result['calc_id']
        return lang

if __name__ == '__main__':


    l = LangID(unk=False)
    l.train()

    if len(sys.argv) > 1:
        text = u' '.join(sys.argv[1:])
    else:
        text = u'hello, world'

    lang = l.classify(text)
    print text, '[ Language:', lang, ']'

{'fraction_non_alphanumeric': 0.13006049325267566, 'fraction_numerical': 0.005118659841786877, 'mean_word_length': 2.2942528735632184, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2431041', 'n_tokens_mistral': 1524, 'n_tokens_neox': 1426, 'n_words': 312}