<commit_before>import re
import math
import os,sys
import pprint
import finalseg
import time

FREQ = {}
total =0.0

def gen_trie(f_name):
	global total
	trie = {}
	content = open(f_name,'rb').read().decode('utf-8')
	for line in content.split("\n"):
		word,freq = line.split(" ")
		freq = float(freq)
		FREQ[word] = freq
		total+=freq
		p = trie
		for c in word:
			if not c in p:
				p[c] ={}
			p = p[c]
		p['']='' #ending flag
	return trie


_curpath=os.path.normpath( os.path.join( os.getcwd(), os.path.dirname(__file__) )  )

print >> sys.stderr, "Building Trie..."
t1 = time.time()
trie = gen_trie(os.path.join(_curpath,"dict.txt"))
FREQ = dict([(k,float(v)/total) for k,v in FREQ.iteritems()]) #normalize
min_freq = min(FREQ.itervalues())
print >> sys.stderr, "loading model cost ", time.time() - t1, "seconds."
print >> sys.stderr, "Trie has been built succesfully."


def __cut_all(sentence):
	N = len(sentence)
	i,j=0,0
	p = trie
	while i<N:
		c = sentence[j]
		if c in p:
			p = p[c]
			if '' in p:
				yield sentence[i:j+1]
			j+=1
			if j>=N:
				i+=1
				j=i
				p=trie
		else:
			p = trie
			i+=1
			j=i


def calc(sentence,DAG,idx,route):
	N = len(sentence)
	route[N] = (1.0,'')
	for idx in xrange(N-1,-1,-1):<commit_msg>use file cache to improve the loading speed after the first time of importing<commit_after>import re
import math
import os,sys
import pprint
import finalseg
import time
import tempfile
import marshal

FREQ = {}
total =0.0

def gen_trie(f_name):
	lfreq = {}
	trie = {}
	ltotal = 0.0
	content = open(f_name,'rb').read().decode('utf-8')
	for line in content.split("\n"):
		word,freq = line.split(" ")
		freq = float(freq)
		lfreq[word] = freq
		ltotal+=freq
		p = trie
		for c in word:
			if not c in p:
				p[c] ={}
			p = p[c]
		p['']='' #ending flag
	return trie, lfreq,ltotal


_curpath=os.path.normpath( os.path.join( os.getcwd(), os.path.dirname(__file__) )  )

print >> sys.stderr, "Building Trie..."
t1 = time.time()
cache_file = os.path.join(tempfile.gettempdir(),"jieba.cache")
load_from_cache_fail = True
if os.path.exists(cache_file) and os.path.getmtime(cache_file)>os.path.getmtime(os.path.join(_curpath,"dict.txt")):
	print >> sys.stderr, "loading model from cache"
	try:
		trie,FREQ,total,min_freq = marshal.load(open(cache_file,'rb'))
		load_from_cache_fail = False
	except:
		load_from_cache_fail = True

if load_from_cache_fail:
	trie,FREQ,total = gen_trie(os.path.join(_curpath,"dict.txt"))
	FREQ = dict([(k,float(v)/total) for k,v in FREQ.iteritems()]) #normalize
	min_freq = min(FREQ.itervalues())
	marshal.dump((trie,FREQ,total,min_freq),open(cache_file,'wb'))

print >> sys.stderr, "loading model cost ", time.time() - t1, "seconds."
print >> sys.stderr, "Trie has been built succesfully."


def __cut_all(sentence):
	N = len(sentence)
	i,j=0,0
	p = trie
	while i<N:
		c = sentence[j]
		if c in p:
			p = p[c]
			if '' in p:
				yield sentence[i:j+1]
			j+=1
			if j>=N:
				i+=1
				j=i
				p=trie
		else:
			p = trie
			i+=1
			j=i


def calc(sentence,DAG,idx,route):
	N = len(sentence)
	route[N] = (1.0,'')
	for idx in xrange(N-1,-1,-1):
{'fraction_non_alphanumeric': 0.16951888924765904, 'fraction_numerical': 0.010978366160800775, 'mean_word_length': 3.715372907153729, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '574874', 'n_tokens_mistral': 1398, 'n_tokens_neox': 1287, 'n_words': 305}