<commit_before>#-*- coding: utf-8 -*-

#Split text file to sentences 
#Extract sentences including topic words.

import sys
from konlpy.tag import Kkma
from konlpy.utils import pprint

reload(sys)
sys.setdefaultencoding('utf-8')

#Split text file to sentences
def SplitTextfile(text_file):
	kkma=Kkma()
	text_data = text_file.read()
	output=kkma.sentences(text_data) 

	#convert unicode to text

	return output

def SentenceExtract(file_name, topic_words):
	
	#file open
	file_name = "data/흥부와놀부.txt"
	fp = open(file_name, 'r')
	
	#split text file to sentences 
	sentences=SplitTextfile(fp)
	print sentences[0], sentences[0].encode('utf8'), sentences[0].decode('utf8')
	
	topic_words=['흥부', '놀부', '제비']
	#make words - sentences matrix
	score=[0 for i in range(len(sentences))]
	#for word in topic_words
		
		

	#<PRINT UNICODE>
	# a = type: unicode
	# a.encode('utf8') -> 한글로 출력!

	fp.close()

SentenceExtract(0,0)<commit_msg>complete sentence extraction by most frequent words<commit_after>#-*- coding: utf-8 -*-

#Split text file to sentences 
#Extract sentences including topic words.

import sys
from konlpy.tag import Kkma
from konlpy.utils import pprint

reload(sys)
sys.setdefaultencoding('utf-8')

#Split text file to sentences
def SplitTextfile(text_file):
	kkma=Kkma()
	text_data = text_file.read()
	#print text_data
	output=kkma.sentences(text_data) 
	#print output[0]

	#convert unicode to text

	return output

def SentenceExtract(file_name, topic_words):
	
	#File open
	file_name = "data/소나기.txt"
	fp = open(file_name, 'r')
	
	#Split text file to sentences 
	sentences=SplitTextfile(fp)
	#print sentences[0], sentences[0].encode('utf8'), sentences[0].decode('utf8')
	
	topic_words=['소년', '소녀', '허수아비','조약돌','스웨터']
	#Make words - sentences matrix
	score=[0 for i in range(len(sentences))]
	for word in topic_words:
		for i in range(len(sentences)):
			#print sentences[i], word, sentences[i].count(word)
			score[i]=score[i]+sentences[i].count(word)

	#print len(sentences), score
		
	#Attatch (sentence, score) and Sorting 
	for i in range(len(sentences)):
		sentences[i]=[sentences[i],score[i]]
	
	sorted_sentences = sorted(sentences, key=lambda l:l[1], reverse=True)

	for i in range(10):
		print sorted_sentences[i][0], sorted_sentences[i][1]

	#<PRINT UNICODE>
	# a = type: unicode
	# a.encode('utf8') -> 한글로 출력!

	fp.close()

SentenceExtract(0,0)
{'fraction_non_alphanumeric': 0.13699788583509515, 'fraction_numerical': 0.011839323467230444, 'mean_word_length': 5.082262210796915, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1317472', 'n_tokens_mistral': 934, 'n_tokens_neox': 922, 'n_words': 219}