<commit_before>#! /usr/bin/env python
# -*- coding=utf-8 -*-

class Bigram:
	'''
	Bigram语言模型
	'''

	#字频表
	character={}
	#词频表
	words={}

	#测试集
	judgeResult=file

	def __init__(self):
		pass

	def trainModel(self, corpusPath='corpus/corpus.txt'):
        with open(corpusPath) as f:
            for line in f:
                line = line.strip()
                for i in range(0, len(line)):
                    if not character.has_key(line[i]):
                        character[line[i]] = 1
                    else:
                        character[line[i]] += 1
                    if not words.has_key(line[i:i+1]):
                        words[line[i:i+1]] = 1
                    else:
                        words[line[i:i+1]] += 1 
                    
        

	def modelOutput(self, characterFilePath='modelData/characterFile.txt', wordsFilePath='modelData/wordsFile.txt'):
        with characterFile = open(characterFilePath, 'w')
        for ch, count in character:
            characterFile.write(ch+':'+count)
        wordsFile = open(wordsFilePath, 'w')
        for word, count in words:
            wordsFile.write(word+':'+count)
        

	def ModelLoader(self, characterFilePath='modelData/characterFile.txt', wordsFilePath='modelData/wordsFile.txt'):
        characterFile = open(characterFilePath, 'r')
        wordsFile = open(wordsFilePath, 'r')
        for line in characterFile:
            parts = line.split(':')
            character[parts[0]] = parts[1]
        for line in wordsFile:
            parts = line.split(':')
            words[parts[0]] = parts[1]

    def getPiecePro(self, piece):
        if character[piece]:<commit_msg>Bigram Temp.<commit_after>#! /usr/bin/env python
# -*- coding=utf-8 -*-

class Bigram:
	'''
	Bigram语言模型
	'''

	#字频表
	character={}
	#词频表
	words={}

	#测试集
	judgeResult=file

	def __init__(self):
		pass

    def trainModel(self, corpusPath='corpus/corpus.txt'):
        with open(corpusPath) as f:
            for line in f:
                line = line.strip()
                for i in range(0, len(line)):
                    if not character.has_key(line[i]):
                        character[line[i]] = 1
                    else:
                        character[line[i]] += 1
                    if not words.has_key(line[i:i+1]):
                        words[line[i:i+1]] = 1
                    else:
                        words[line[i:i+1]] += 1 

	def modelOutput(self, characterFilePath='modelData/characterFile.txt', wordsFilePath='modelData/wordsFile.txt'):
        with characterFile = open(characterFilePath, 'w')
        for ch, count in character:
            characterFile.write(ch+':'+count)
        wordsFile = open(wordsFilePath, 'w')
        for word, count in words:
            wordsFile.write(word+':'+count)
        

	def ModelLoader(self, characterFilePath='modelData/characterFile.txt', wordsFilePath='modelData/wordsFile.txt'):
        characterFile = open(characterFilePath, 'r')
        wordsFile = open(wordsFilePath, 'r')
        for line in characterFile:
            parts = line.split(':')
            character[parts[0]] = parts[1]
        for line in wordsFile:
            parts = line.split(':')
            words[parts[0]] = parts[1]

    def getPiecePro(self, piece):
        if character[piece]:
{'fraction_non_alphanumeric': 0.12908101571946795, 'fraction_numerical': 0.007859733978234583, 'mean_word_length': 1.887434554973822, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3150396', 'n_tokens_mistral': 1018, 'n_tokens_neox': 974, 'n_words': 208}