<jupyter_start><jupyter_text>Topic Modelling - All text<jupyter_code>df['all_text']=df['Outcome: SM']+' '+df['Support']+' '+df['Outcome: Overall Verdict']
# what is SORA?
import gensim
docs_words = []
for text in df['all_text'].dropna():
    docs_words.append(gensim.utils.simple_preprocess(text, deacc=True))
ids = df['all_text'].dropna().index
len(docs_words)
# Define functions for stopwords, bigrams, trigrams and lemmatization
def remove_stopwords(texts):
    return [[word for word in simple_preprocess(str(doc)) if word not in stop_words] for doc in texts]

def make_bigrams(texts):
    return [bigram_mod[doc] for doc in texts]

def lemmatization(texts, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV']):
    """https://spacy.io/api/annotation"""
    texts_out = []
    for sent in texts:
        doc = nlp(" ".join(sent)) 
        texts_out.append([token.lemma_ for token in doc if token.pos_ in allowed_postags])
    return texts_out

stop_words = stopwords.words('english')
# stop_words.extend(['facebook','myspace','defendant','defendent','complainant','st','paul','post','posting','evidence','court','video','photo','photograph'])
stop_words.extend(['defendant','defendent','complainant'])

# Build the bigram and trigram models
bigram = gensim.models.Phrases(docs_words, min_count=5, threshold=100) # higher threshold fewer phrases. 

# Faster way to get a sentence clubbed as a trigram/bigram
bigram_mod = gensim.models.phrases.Phraser(bigram)


# Remove Stop Words
docs_words_nostops = remove_stopwords(docs_words)

# Form Bigrams
data_words_bigrams = make_bigrams(docs_words_nostops)

# Initialize spacy 'en' model, keeping only tagger component (for efficiency)
# python3 -m spacy download en
nlp = spacy.load('en', disable=['parser', 'ner'])

# Do lemmatization keeping only noun, adj, vb, adv
# data_lemmatized = lemmatization(docs_words_nostops, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV'])
data_lemmatized = lemmatization(data_words_bigrams, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV'])

# Create Dictionary
id2word = corpora.Dictionary(data_lemmatized)

# Create Corpus
texts = data_lemmatized

# Term Document Frequency
corpus = [id2word.doc2bow(text) for text in texts]
mallet_path = '/Users/rakshitanagalla/Desktop/SSW-Legal/mallet-2.0.8/bin/mallet'
def compute_coherence_values(dictionary, corpus, texts, limit, start=2, step=3):
    """
    Compute c_v coherence for various number of topics
    For details, refer: 

    Parameters:
    ----------
    dictionary : Gensim dictionary
    corpus : Gensim corpus
    texts : List of input texts
    limit : Max num of topics

    Returns:
    -------
    model_list : List of LDA topic models
    coherence_values : Coherence values corresponding to the LDA model with respective number of topics
    """
    coherence_values = []
    model_list = []
    for num_topics in range(start, limit, step):
        model = gensim.models.wrappers.LdaMallet(mallet_path, corpus=corpus, num_topics=num_topics, id2word=dictionary)
        model_list.append(model)
        coherencemodel = CoherenceModel(model=model, texts=texts, dictionary=dictionary, coherence='c_v') # try 'u_mass' too
        coherence_values.append(coherencemodel.get_coherence())

    return model_list, coherence_values
# Can take a long time to run.
model_list, coherence_values = compute_coherence_values(dictionary=id2word, corpus=corpus, texts=data_lemmatized, start=2, limit=8, step=1)
# Show graph
# limit=12; start=2; step=2;
limit=8; start=2; step=1;
x = range(start, limit, step)
plt.plot(x, coherence_values)
plt.xlabel("Num Topics")
plt.ylabel("Coherence score")
plt.legend(("coherence_values"), loc='best')
plt.show()
# Print the coherence scores
for m, cv in zip(x, coherence_values):
    print("Num Topics =", m, " has Coherence Value of", round(cv, 4))
optimal_model = model_list[2]
model_topics = optimal_model.show_topics(formatted=False)
pprint(optimal_model.print_topics(num_words=10))<jupyter_output>[(0,
  '0.058*"evidence" + 0.037*"witness" + 0.037*"motion" + 0.037*"photo" + '
  '0.026*"photograph" + 0.026*"deny" + 0.021*"include" + 0.018*"criminal" + '
  '0.018*"record" + 0.018*"possession"'),
 (1,
  '0.090*"facebook" + 0.037*"post" + 0.018*"page" + 0.018*"friend" + '
  '0.018*"order" + 0.018*"people" + 0.016*"staff" + 0.013*"conduct" + '
  '0.013*"student" + 0.013*"specific"'),
 (2,
  '0.051*"court" + 0.034*"video" + 0.032*"message" + 0.032*"evidence" + '
  '0.024*"testimony" + 0.024*"trial" + 0.024*"juror" + 0.022*"victim" + '
  '0.022*"make" + 0.020*"social"'),
 (3,
  '0.039*"myspace" + 0.039*"year" + 0.036*"sentence" + 0.031*"account" + '
  '0.025*"claim" + 0.022*"sex" + 0.017*"offender" + 0.017*"term" + '
  '0.017*"give" + 0.017*"count"')]<jupyter_text>Visualize Keywords<jupyter_code>state_file = optimal_model.fstate()
print(state_file)
from process_pyldaviz import get_pyldaviz
lda_prop = get_pyldaviz(state_file)
vis_data = pyLDAvis.prepare(**lda_prop)
pyLDAvis.display(vis_data)<jupyter_output>/var/folders/8g/1g347k696y56fdzqtj5x54cr0000gn/T/70e63c_state.mallet.gz
[12.5, 12.5, 12.5, 12.5], 0.01<jupyter_text>Dominant topic in each of the cases<jupyter_code>data = [' '.join(doc) for doc in docs_words]

def format_topics_sentences(ldamodel, corpus=corpus, texts=data):
    # Init output
    sent_topics_df = pd.DataFrame()

    # Get main topic in each document
    for i, row in enumerate(ldamodel[corpus]):
        row = sorted(row, key=lambda x: (x[1]), reverse=True)
        # Get the Dominant topic, Perc Contribution and Keywords for each document
        for j, (topic_num, prop_topic) in enumerate(row):
            if j == 0:  # => dominant topic
                wp = ldamodel.show_topic(topic_num)
                topic_keywords = ", ".join([word for word, prop in wp])
                sent_topics_df = sent_topics_df.append(pd.Series([int(topic_num), round(prop_topic,4), topic_keywords]), ignore_index=True)
            else:
                break
    sent_topics_df.columns = ['Dominant_Topic', 'Perc_Contribution', 'Topic_Keywords']

    # Add original text to the end of the output
    contents = pd.Series(texts)
    sent_topics_df = pd.concat([sent_topics_df, contents], axis=1)
    return(sent_topics_df)


df_topic_sents_keywords = format_topics_sentences(ldamodel=optimal_model, corpus=corpus, texts=data)

# Format
df_dominant_topic = df_topic_sents_keywords.reset_index()
df_dominant_topic.columns = ['Document_No', 'Dominant_Topic', 'Topic_Perc_Contrib', 'Keywords', 'Text']

# Show
# df_dominant_topic.head(10)
df_dominant_topic['Case Name'] = list(df.dropna(subset=['all_text']).index)
df_dominant_topic.sort_values(by=['Topic_Perc_Contrib'], ascending=False)[['Case Name','Dominant_Topic','Topic_Perc_Contrib','Text','Keywords']].head(5)
df_dominant_topic.Dominant_Topic.value_counts().plot(kind='bar')
plt.hist(df_dominant_topic['Topic_Perc_Contrib']*100)
a=pd.merge(df_dominant_topic, df[['Platform','Evidence Type','Offence']].reset_index(), on = 'Case Name').groupby('Dominant_Topic')['Evidence Type'].unique()
a
a[1]<jupyter_output><empty_output><jupyter_text>Most representative document for each topic<jupyter_code># Group top 5 sentences under each topic

df_topic_sents_keywords['Case Name'] = list(df.dropna(subset=['all_text']).index)

sent_topics_sorteddf_mallet = pd.DataFrame()

sent_topics_outdf_grpd = df_topic_sents_keywords.groupby('Dominant_Topic')

for i, grp in sent_topics_outdf_grpd:
    sent_topics_sorteddf_mallet = pd.concat([sent_topics_sorteddf_mallet, 
                                             grp.sort_values(['Perc_Contribution'], ascending=[0]).head(1)], 
                                            axis=0)

# Reset Index    
sent_topics_sorteddf_mallet.reset_index(drop=True, inplace=True)

# Format
sent_topics_sorteddf_mallet.columns = ['Topic_Num', "Topic_Perc_Contrib", "Keywords", "Text",'Case Name']

# Show
sent_topics_sorteddf_mallet[['Topic_Num',"Topic_Perc_Contrib",'Case Name',"Keywords", "Text"]]#.head()<jupyter_output><empty_output><jupyter_text>Topic distribution across documents<jupyter_code># Number of Documents for Each Topic
topic_counts = df_topic_sents_keywords['Dominant_Topic'].value_counts()

# Percentage of Documents for Each Topic
topic_contribution = round(topic_counts/topic_counts.sum(), 4)

# Topic Number and Keywords
topic_num_keywords = sent_topics_sorteddf_mallet[['Topic_Num', 'Keywords']]

# Concatenate Column wise
df_topics = pd.concat([topic_num_keywords, topic_counts, topic_contribution], axis=1)

# Change Column names
df_topics.columns = ['Dominant_Topic', 'Topic_Keywords', 'Num_Documents', 'Perc_Documents']

# Show
df_topics<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14193324864303036, 'fraction_numerical': 0.028409747083958888, 'mean_word_length': 5.27536231884058, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 20, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '274623', 'license': 'Apache-2.0', 'n_tokens_mistral': 3320, 'n_tokens_neox': 3054, 'n_words': 596, 'path': 'annotated-topic-model.ipynb', 'repo_name': 'rakshita95/court-document-ananlysis', 'size': '8659'}