<commit_before>#!/usr/bin/env python3
#一個統計頻率的程式
#https://github.com/rime/home/issues/18

from collections import defaultdict

d = defaultdict(int)
enc = ""

def freq_sum(encoding):
  try:
    d.clear()
    f = open("input.txt", encoding=encoding) #輸入文件
    for line in f:
      line = line.strip()
      if line:
        fs = line.rsplit("\t", 1)
        if len(fs) == 2 and fs[1].isdigit():#詞\t音\t頻 或 詞\t頻
          word, freq = fs
        else:
          word, freq = line, 1 ##詞\t音 或 詞，默認詞頻爲1
        d[word] += int(freq) #累加
    f.close()
    global enc
    enc = encoding
    return True
  except:
    return False

any(map(freq_sum, ["utf-8-sig", "utf-16", "gbk", "gb18030"]))
print("file encoding:", enc)
l = sorted(d.items(), key = lambda x: x[1], reverse = True) #排序
t = open("output.txt", "w", encoding = enc) #輸出文件
for word, freq in l:
  print("%s\t%d" % (word, freq), file = t) #輸出

t.close()
<commit_msg>pylint<commit_after>#!/usr/bin/env python3
"""一個統計頻率的程式
https://github.com/rime/home/issues/18
"""

from collections import defaultdict

ENC = "utf-8"
ENCODINGS = ["utf-8-sig", "utf-16", "gbk", "gb18030"]
INPUT_NAME = "input.txt"
OUTPUT_NAME = "output.txt"

def write_dic(dic):
    "保存文件"
    lists = sorted(dic.items(), key=lambda x: x[1], reverse=True) #排序
    target_file = open(OUTPUT_NAME, "w", encoding=ENC) #輸出文件
    for word, freq in lists:
        print("%s\t%d" % (word, freq), file=target_file) #輸出
    target_file.close()

def try_enc(enc):
    "嘗試指定編碼"
    try:
        dic = defaultdict(int)
        input_file = open(INPUT_NAME, encoding=enc) #輸入文件
        for line in input_file:
            line = line.strip()
            if line:
                fields = line.rsplit("\t", 1)
                if len(fields) == 2 and fields[1].isdigit():#詞\t音\t頻 或 詞\t頻
                    word, freq = fields
                else:
                    word, freq = line, 1 #詞\t音 或 詞，默認詞頻爲1
                dic[word] += int(freq) #累加
        input_file.close()
        write_dic(dic)
        return True
    except UnicodeError:
        return False

def main():
    "嘗試編碼列表"
    for enc in ENCODINGS:
        if try_enc(enc):
            break

if __name__ == "__main__":
    main()

{'fraction_non_alphanumeric': 0.15485921889191645, 'fraction_numerical': 0.01589464123524069, 'mean_word_length': 2.3078078078078077, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1395444', 'n_tokens_mistral': 925, 'n_tokens_neox': 907, 'n_words': 205}