<jupyter_start><jupyter_text>(8) 1.	Asbest (oranje netwerk, 14 uitspraken);(4) 2.	Zorgplichtbegrip, inlening, eigen schuld en ‘grove schuld’ onder art. 7A:1638x BW (lichtblauwe cluster, 22 uitspraken), welk onderwerp verder zou kunnen worden uitgesplitst in eigen schuld en ‘grove schuld’ enerzijds en het zorgplichtbegrip en inlening anderzijds;(0) 3.	Causaal verband (blauw-paarse cluster, 11 uitspraken);(2) 4.	Verzekeringsplicht en woon-werkverkeer (blauwe cluster, 27 uitspraken);(3?) 5.	Omkeringsregel algemeen (rode cluster, 6 uitspraken);(?) 6.	Deskundigen, stelplicht en bewijslast (lichtblauwe cluster, 6 uitspraken).<jupyter_code>from IPython.display import display

relevances_dict = {}
for cl in sorted(nodes_df.community.unique()):
    nodes_in_cluster = [i for i, node in enumerate(network['nodes']) if node['community']==str(cl)]
    nodes_with_text = nodes_df[nodes_df['community']==cl]
    print(cl, len(nodes_in_cluster), len(nodes_with_text))
    relevances = get_relevances(wordcounts, term_indices, cl, nodes_df.community)
    if relevances is not None:
        relevances = relevances.sort_values(['p_logodds', 'pw_logodds'], ascending=False)
        relevances_dict[cl] = relevances
        display(relevances.head(10))
import os 

output_path = '/media/sf_VBox_Shared/CaseLaw/graphs/clusters_text/wgaans/'
for i in relevances_dict:
    relevances_dict[i].to_csv(os.path.join(output_path, 'cl{}.csv'.format(i)))
relevances_dict[4].plot('cl_cnt', 'p_logodds', kind='scatter')
import matplotlib.pyplot as plt

relevances = relevances_dict[0]
relevances.plot('p_cl', 'p_co', kind='scatter', figsize=(20,20))
for (l, x, y) in relevances[['p_cl', 'p_co']].itertuples():
    if x > y:
        plt.annotate(l, xy=(x, y))
plt.show()<jupyter_output><empty_output><jupyter_text>Topic models<jupyter_code>dictionary = gensim.corpora.Dictionary(nodes_df['tokenized'])
print(dictionary)
min_count = 2
no_above = 0.8
dictionary.filter_extremes(no_below=min_count, no_above=no_above)
dictionary.compactify() 
print(dictionary)
corpus = [dictionary.doc2bow(t) for t in nodes_df['tokenized']]
num_topics = len(relevances_dict.keys())
ldamodel = gensim.models.ldamodel.LdaModel(corpus, num_topics=num_topics, passes=10, id2word=dictionary)
corpus_lda = ldamodel[corpus]
ldamodel.print_topics()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12966938600257621, 'fraction_numerical': 0.017604121940747102, 'mean_word_length': 6.6644736842105265, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '414878', 'license': 'Apache-2.0', 'n_tokens_mistral': 936, 'n_tokens_neox': 901, 'n_words': 177, 'path': 'clusters.ipynb', 'repo_name': 'dafnevk/case-law-semantics', 'size': '2329'}