<commit_before>from __future__ import with_statement
__version__ = '0.31'
__license__ = 'MIT'

import re
import os
import sys
import finalseg
import time
import tempfile
import marshal
from math import log
import random
import threading
from functools import wraps

DICTIONARY = "dict.txt"
DICT_LOCK = threading.RLock()
trie = None # to be initialized
FREQ = {}
min_freq = 0.0
total =0.0
user_word_tag_tab={}
initialized = False

def gen_trie(f_name):
    lfreq = {}
    trie = {}
    ltotal = 0.0
    with open(f_name, 'rb') as f:
        lineno = 0 
        for line in f.read().rstrip().decode('utf-8').split('\n'):
            lineno += 1
            try:
                word,freq,_ = line.split(' ')
                freq = float(freq)
                lfreq[word] = freq
                ltotal+=freq
                p = trie
                for c in word:
                    if c not in p:
                        p[c] ={}
                    p = p[c]
                p['']='' #ending flag
            except ValueError, e:
                print >> sys.stderr, f_name, ' at line', lineno, line
                raise e
    return trie, lfreq,ltotal

def initialize(*args):
    global trie, FREQ, total, min_freq, initialized
    if len(args)==0:
        dictionary = DICTIONARY
    else:
        dictionary = args[0]
    with DICT_LOCK:
        if initialized:
            return
        if trie:
            del trie
            trie = None
        _curpath=os.path.normpath( os.path.join( os.getcwd(), os.path.dirname(__file__) )  )

        abs_path = os.path.join(_curpath,dictionary)
        print >> sys.stderr, "Building Trie..., from " + abs_path
        t1 = time.time()
        if abs_path == os.path.join(_curpath,"dict.txt"): #defautl dictionary
            cache_file = os.path.join(tempfile.gettempdir(),"jieba.cache")
        else: #customer dictionary
            cache_file = os.path.join(tempfile.gettempdir(),"jieba.user."+str(hash(abs_path))+".cache")

        load_from_cache_fail = True
        if os.path.exists(cache_file) and os.path.getmtime(cache_file)>os.path.getmtime(abs_path):
            print >> sys.stderr, "loading model from cache " + cache_file
            try:
                trie,FREQ,total,min_freq = marshal.load(open(cache_file,'rb'))
                load_from_cache_fail = False
            except:
                load_from_cache_fail = True

        if load_from_cache_fail:
            trie,FREQ,total = gen_trie(abs_path)
            FREQ = dict([(k,log(float(v)/total)) for k,v in FREQ.iteritems()]) #normalize
            min_freq = min(FREQ.itervalues())
            print >> sys.stderr, "dumping model to file cache " + cache_file
            try:
                tmp_suffix = "."+str(random.random())
                with open(cache_file+tmp_suffix,'wb') as temp_cache_file:
                    marshal.dump((trie,FREQ,total,min_freq),temp_cache_file)
                if os.name=='nt':
                    import shutil
                    replace_file = shutil.move
                else:
                    replace_file = os.rename
                replace_file(cache_file+tmp_suffix,cache_file)
            except:
                print >> sys.stderr, "dump cache file failed."
                import traceback
                print >> sys.stderr, traceback.format_exc()

        initialized = True

        print >> sys.stderr, "loading model cost ", time.time() - t1, "seconds."
        print >> sys.stderr, "Trie has been built succesfully."


def require_initialized(fn):

    @wraps(fn)
    def wrapped(*args, **kwargs):
	global initialized
        if initialized:
            return fn(*args, **kwargs)
        else:
            initialize(DICTIONARY)
            return fn(*args, **kwargs)

    return wrapped


def __cut_all(sentence):
    dag = get_DAG(sentence)
    old_j = -1
    for k,L in dag.iteritems():
        if len(L)==1 and k>old_j:
            yield sentence[k:L[0]+1]
            old_j = L[0] 
        else:
            for j in L:
                if j>k:
                    yield sentence[k:j+1]
                    old_j = j


def calc(sentence,DAG,idx,route):
    N = len(sentence)
    route[N] = (0.0,'')<commit_msg>use logging instead of print<commit_after>from __future__ import with_statement
__version__ = '0.31'
__license__ = 'MIT'

import re
import os
import sys
import finalseg
import time
import tempfile
import marshal
from math import log
import random
import threading
from functools import wraps
import logging

DICTIONARY = "dict.txt"
DICT_LOCK = threading.RLock()
trie = None # to be initialized
FREQ = {}
min_freq = 0.0
total =0.0
user_word_tag_tab={}
initialized = False
logger = logging.getLogger(__name__)

def gen_trie(f_name):
    lfreq = {}
    trie = {}
    ltotal = 0.0
    with open(f_name, 'rb') as f:
        lineno = 0 
        for line in f.read().rstrip().decode('utf-8').split('\n'):
            lineno += 1
            try:
                word,freq,_ = line.split(' ')
                freq = float(freq)
                lfreq[word] = freq
                ltotal+=freq
                p = trie
                for c in word:
                    if c not in p:
                        p[c] ={}
                    p = p[c]
                p['']='' #ending flag
            except ValueError, e:
                logger.debug('%s at line %s %s' % (f_name,  lineno, line))
                raise ValueError, e
    return trie, lfreq,ltotal

def initialize(*args):
    global trie, FREQ, total, min_freq, initialized
    if len(args)==0:
        dictionary = DICTIONARY
    else:
        dictionary = args[0]
    with DICT_LOCK:
        if initialized:
            return
        if trie:
            del trie
            trie = None
        _curpath=os.path.normpath( os.path.join( os.getcwd(), os.path.dirname(__file__) )  )

        abs_path = os.path.join(_curpath,dictionary)
        logger.debug("Building Trie..., from %s" % abs_path)
        t1 = time.time()
        if abs_path == os.path.join(_curpath,"dict.txt"): #defautl dictionary
            cache_file = os.path.join(tempfile.gettempdir(),"jieba.cache")
        else: #customer dictionary
            cache_file = os.path.join(tempfile.gettempdir(),"jieba.user."+str(hash(abs_path))+".cache")

        load_from_cache_fail = True
        if os.path.exists(cache_file) and os.path.getmtime(cache_file)>os.path.getmtime(abs_path):
            logger.debug("loading model from cache %s" % cache_file)
            try:
                trie,FREQ,total,min_freq = marshal.load(open(cache_file,'rb'))
                load_from_cache_fail = False
            except:
                load_from_cache_fail = True

        if load_from_cache_fail:
            trie,FREQ,total = gen_trie(abs_path)
            FREQ = dict([(k,log(float(v)/total)) for k,v in FREQ.iteritems()]) #normalize
            min_freq = min(FREQ.itervalues())
            logger.debug("dumping model to file cache %s" % cache_file)
            try:
                tmp_suffix = "."+str(random.random())
                with open(cache_file+tmp_suffix,'wb') as temp_cache_file:
                    marshal.dump((trie,FREQ,total,min_freq),temp_cache_file)
                if os.name=='nt':
                    import shutil
                    replace_file = shutil.move
                else:
                    replace_file = os.rename
                replace_file(cache_file+tmp_suffix,cache_file)
            except:
                logger.error("dump cache file failed.")
                logger.exception("")

        initialized = True

        logger.debug("loading model cost %s seconds." % (time.time() - t1))
        logger.debug("Trie has been built succesfully.")


def require_initialized(fn):

    @wraps(fn)
    def wrapped(*args, **kwargs):
        global initialized
        if initialized:
            return fn(*args, **kwargs)
        else:
            initialize(DICTIONARY)
            return fn(*args, **kwargs)

    return wrapped


def __cut_all(sentence):
    dag = get_DAG(sentence)
    old_j = -1
    for k,L in dag.iteritems():
        if len(L)==1 and k>old_j:
            yield sentence[k:L[0]+1]
            old_j = L[0] 
        else:
            for j in L:
                if j>k:
                    yield sentence[k:j+1]
                    old_j = j


def calc(sentence,DAG,idx,route):
    N = len(sentence)
    route[N] = (0.0,'')
{'fraction_non_alphanumeric': 0.11133579160223624, 'fraction_numerical': 0.005709527774473653, 'mean_word_length': 1.809221516872703, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7440712', 'n_tokens_mistral': 2668, 'n_tokens_neox': 2555, 'n_words': 609}