<jupyter_start><jupyter_text>获取索引<jupyter_code>text1.concordance('monstrous')
text1.similar('monstrous')
# 两个词相同的上下文
text2.common_contexts(['monstrous', 'very'])
# 获取一个词在文本中的位置的离散图
text4.dispersion_plot(['citizen', 'democracy', 'freedom', 'duties', 'America'])

len(set(text3))
def lexical_diversity(text):
    """每个词出现的平均频率"""
    return len(text) / len(set(text))


def percentage(count, total):
    return 100 * count / total

lexical_diversity(text4)
sent2
lexical_diversity(sent2)
# 频率分布
fdist = FreqDist(text1)
fdist.most_common(10)
# 低频词， 只出现了一次的词
len(fdist.hapaxes()), fdist[fdist.hapaxes()[0]]
# 找出文本词汇表长度中超过 15 个字符的词
V = set(text1)
long_words = [w for w in V if len(w) > 15]
long_words
# 聊天语料库中所有长度超过7 个字符出现次数超过 7 次的词
fdist5 = FreqDist(text5)
sorted([word for word in set(text5) if len(word) > 7 and fdist5[word] > 7])<jupyter_output><empty_output><jupyter_text>双连词(bigrams)<jupyter_code>list(bigrams('more is said than done'.split()))<jupyter_output><empty_output><jupyter_text>语言的固定搭配一般就是频率较高的连词<jupyter_code>text4.collocations()
# 计算词的长度分布
fdist_len = FreqDist([len(word) for word in text1])
fdist_len
sent7<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14521739130434783, 'fraction_numerical': 0.026956521739130435, 'mean_word_length': 8.134920634920634, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '621991', 'license': 'MIT', 'n_tokens_mistral': 575, 'n_tokens_neox': 599, 'n_words': 81, 'path': 'natual_lang_processing/nltk-sample.ipynb', 'repo_name': 'aollio/toys', 'size': '1150'}