<commit_before>#encoding=utf-8
import jieba
import os
try:
    from analyzer import ChineseAnalyzer
except ImportError:
    pass

_curpath = os.path.normpath( os.path.join( os.getcwd(), os.path.dirname(__file__) )  )
abs_path = os.path.join(_curpath, "idf.txt")

IDF_DICTIONARY = abs_path
STOP_WORDS = set([
    "the","of","is","and","to","in","that","we","for","an","are","by","be","as","on","with","can","if","from","which","you","it","this","then","at","have","all","not","one","has","or","that"
])

def set_idf_path(idf_path):
    global IDF_DICTIONARY
    abs_path = os.path.normpath( os.path.join( os.getcwd(), idf_path )  )
    if not os.path.exists(abs_path):
        raise Exception("jieba: path does not exist:" + abs_path)
    IDF_DICTIONARY = abs_path
    return

def get_idf(abs_path):
    content = open(abs_path,'rb').read().decode('utf-8')
    idf_freq = {}
    lines = content.split('\n')
    for line in lines:
        word,freq = line.split(' ')
        idf_freq[word] = float(freq)<commit_msg>fix issues (https://github.com/fxsjy/jieba/issues/125)<commit_after>#encoding=utf-8
import jieba
import os
try:
    from analyzer import ChineseAnalyzer
except ImportError:
    pass

_curpath = os.path.normpath( os.path.join( os.getcwd(), os.path.dirname(__file__) )  )
abs_path = os.path.join(_curpath, "idf.txt")

IDF_DICTIONARY = abs_path
STOP_WORDS = set([
    "the","of","is","and","to","in","that","we","for","an","are","by","be","as","on","with","can","if","from","which","you","it","this","then","at","have","all","not","one","has","or","that"
])

def set_idf_path(idf_path):
    global IDF_DICTIONARY
    abs_path = os.path.normpath( os.path.join( os.getcwd(), idf_path )  )
    if not os.path.exists(abs_path):
        raise Exception("jieba: path does not exist:" + abs_path)
    IDF_DICTIONARY = abs_path
    return

def get_idf(abs_path):
    content = open(abs_path,'rb').read().decode('utf-8')
    idf_freq = {}
    lines = content.split('\n')
    if lines and not lines[-1]:
        lines.pop(-1)
    for line in lines:
        word,freq = line.split(' ')
        idf_freq[word] = float(freq)
{'fraction_non_alphanumeric': 0.20179160773220178, 'fraction_numerical': 0.004243281471004243, 'mean_word_length': 4.345088161209068, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1429061', 'n_tokens_mistral': 788, 'n_tokens_neox': 783, 'n_words': 133}