<jupyter_start><jupyter_text>......等等，他们为什么要分我们的地瓜<jupyter_code>from collections import Counter
import pkuseg
content = []
with open("words.txt", encoding="utf-8") as f:
    content = f.read()

seg = pkuseg.pkuseg(model_name='./ctb8')
text = seg.cut(content)


stopwords = []

with open("stopwords.txt", encoding="utf-8") as f:
    stopwords = f.read()

new_text = []

for w in text:
    if w not in stopwords:
        new_text.append(w)


counter = Counter(new_text)
print(counter)<jupyter_output>loading model
finish
Counter({'游戏': 11, '朋友': 3, '外挂': 3, '挺': 2, '称号': 2, '分数': 2, '跳': 2, '微信': 2, '款': 2, '刚刚': 1, '画面': 1, '水平': 1, '紧张': 1, '分曾': 1, '打到': 1, '6000多': 1, '厉害': 1, '时间': 1, '好玩': 1, '无聊': 1, '大师': 1, '上个星期': 1, '6000分': 1, '发现': 1, '立地成佛': 1, '小心': 1, '佛系': 1, '玩': 1, '排行榜': 1, '修改': 1, '数据库': 1, '引发': 1, '思考': 1, '当成': 1, 'Demo': 1, '版本': 1, '体现': 1, '程序': 1, '平台': 1, '威力': 1, '匆忙': 1, '简单': 1, '发布': 1, '效果': 1, '超出': 1, '预期': 1, '我们自己': 1, '开玩笑': 1, '有史以来': 1, '用户': 1, '规模': 1, 'DAU': 1, '1点几亿': 1, '想到': 1, '朋友圈': 1, '打出': 1})
{'fraction_non_alphanumeric': 0.2889733840304182, 'fraction_numerical': 0.0655893536121673, 'mean_word_length': 4.014285714285714, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '578713', 'license': 'MIT', 'n_tokens_mistral': 640, 'n_tokens_neox': 602, 'n_words': 154, 'path': 'separate-word.ipynb', 'repo_name': 'cyneck/separate-words', 'size': '1052'}