<jupyter_start><jupyter_text>Train LDA models with various number of topics for testing purposes<jupyter_code># Train LDA model on articles
parser = LDA_extractor.LDA_parser(articles,
                                  language='english', 
                                  preprocessor_type='spacy',
                                  num_topics = 32, 
                                  passes = 100,
                                  custom_filter = ['said'])

parser.print_topics(words_per_topic = 10) 
topic_mixtures = parser.extract_topics(max_words_per_topic=50, threshold=0.005)
print(topic_mixtures)

# extract topics as a dictionary
topics = parser.extract_topic_words(max_words_per_topic=50, threshold=0.005)
print(topics)
# Train LDA model on articles
parser100 = LDA_extractor.LDA_parser(articles,
                                  language='english', 
                                  preprocessor_type='spacy',
                                  num_topics = 100, 
                                  passes = 100,
                                  custom_filter = ['said'])

parser100.print_topics(words_per_topic = 20)
topic_mixtures = parser100.extract_topics(max_words_per_topic=50, threshold=0.005)
print(topic_mixtures)

# extract topics as a dictionary
topics = parser100.extract_topic_words(max_words_per_topic=50, threshold=0.005)
print(topics)
# Train LDA model on articles
parser50 = LDA_extractor.LDA_parser(articles,
                                  language='english', 
                                  preprocessor_type='spacy',
                                  num_topics = 100, 
                                  passes = 100,
                                  custom_filter = ['said'])

parser50.print_topics(words_per_topic = 20)
topic_mixtures = parser50.extract_topics(max_words_per_topic=50, threshold=0.005)
print(topic_mixtures)

# extract topics as a dictionary
topics = parser100.extract_topic_words(max_words_per_topic=50, threshold=0.005)
print(topics)
# Train LDA model on articles
parser50a = LDA_extractor.LDA_parser(articles,
                                  language='english', 
                                  preprocessor_type='spacy',
                                  num_topics = 50, 
                                  passes = 100,
                                  custom_filter = ['said'])

parser50a.print_topics(words_per_topic = 20)
topic_mixtures = parser50a.extract_topics(max_words_per_topic=50, threshold=0.005)
print(topic_mixtures)

# extract topics as a dictionary
topics = parser50a.extract_topic_words(max_words_per_topic=50, threshold=0.005)
print(topics)
# Train LDA model on articles
parser64 = LDA_extractor.LDA_parser(articles,
                                  language='english',
                                  preprocessor_type='spacy',
                                  num_topics = 64,
                                  passes = 100,
                                  custom_filter = ['said'])

parser64.print_topics(words_per_topic = 20)
topic_mixtures = parser64.extract_topics(max_words_per_topic=50, threshold=0.005)
print(topic_mixtures)

# extract topics as a dictionary
topics = parser64.extract_topic_words(max_words_per_topic=50, threshold=0.005)
print(topics)
# Train LDA model on articles
parser75 = LDA_extractor.LDA_parser(articles,
                                  language='english',
                                  preprocessor_type='spacy',
                                  num_topics = 75,
                                  passes = 100,
                                  custom_filter = ['said'])

parser75.print_topics(words_per_topic = 20)
topic_mixtures = parser75.extract_topics(max_words_per_topic=50, threshold=0.005)
print(topic_mixtures)

# extract topics as a dictionary
topics = parser75.extract_topic_words(max_words_per_topic=50, threshold=0.005)
print(topics)<jupyter_output>Initializing model...

spaCy preprocessor selected.
Fitting LDA topic modelling...
Preprocessing corpus...
Creating corpora dictionary...
Translating doc2bow corpus...
Running LDA...

Done in 568.856 seconds.
(39, '0.033*"anwar" + 0.030*"malaysia" + 0.025*"said" + 0.019*"mahathir" + 0.013*"minister" + 0.011*"asia" + 0.011*"president" + 0.010*"ibrahim" + 0.010*"pacific" + 0.010*"malaysian" + 0.009*"prime" + 0.009*"habibie" + 0.008*"leaders" + 0.008*"meeting" + 0.007*"estrada" + 0.007*"arrest" + 0.007*"country" + 0.007*"charges" + 0.007*"people" + 0.007*"capital"')
(35, '0.014*"turkey" + 0.013*"mob" + 0.011*"said" + 0.011*"leader" + 0.011*"oz" + 0.008*"he" + 0.008*"italian" + 0.008*"wanted" + 0.008*"calascibetta" + 0.008*"ocalan" + 0.008*"held" + 0.008*"hostage" + 0.008*"reppas" + 0.008*"harboring" + 0.007*"denied" + 0.006*"turkish" + 0.006*"monday" + 0.005*"imprisoned" + 0.005*"stability" + 0.005*"pressure"')
(72, '0.019*"dlrs" + 0.017*"cardoso" + 0.013*"spending" + 0.012*"workers" + 0.[...]<jupyter_text>Create word embedding models<jupyter_code># Sentence tokenize t
import nltk.data
import preprocessor3

# Compound all docs into a megadocument to process
megadoc = ""
for set_id in corpus:
    for article in corpus[set_id]['articles']:
        megadoc += article
sent_detector = nltk.data.load('tokenizers/punkt/english.pickle')
pre = preprocessor3.spacy_preprocessor()
megadoc_sents = sent_detector.tokenize(megadoc)
embeddings_corpus = pre.preprocess_texts(megadoc_sents, tags=[])
model = Word2Vec(embeddings_corpus, size=400, window=10, iter=100, negative=10, sg=1, min_count=0, workers=-1) 
model2 = Word2Vec(common_texts, size=100, window=5, min_count=1, workers=-1)<jupyter_output><empty_output><jupyter_text>Evaluate Rouge For C_CBOW use sg=0, size=200, sim_theshold=0.93For C_SKIP use sg=1, size=400, sim_theshold=0.94(So results are comparable to paper)<jupyter_code>from centroid_word_embeddings33 import CentroidWordEmbeddingsSummarizer
summarizer = CentroidWordEmbeddingsSummarizer(model, parser = parser64, sim_threshold = 0.94)
corpus[list(corpus.keys())[4]]['articles']
SET_ID = 4
compound_article = ''.join(corpus[list(corpus.keys())[SET_ID]]['articles'])
print(summarizer32.summarize(compound_article))
import rouge_evaluator3
hypos = []
refs = []
for set_id in corpus.keys():
    hypos.append(summarizer32.summarize(''.join(corpus[set_id]['articles'])))
    refs.append(corpus[set_id]['summaries'])
print(rouge_evaluator3.evaluate_hypotheses(hypos, refs))<jupyter_output>Evaluation with Avg
	metric:	P: 33.58	R: 33.77	F1: 33.67
	metric:	P:  6.72	R:  6.77	F1:  6.74
	metric:	P:  2.13	R:  2.15	F1:  2.14
	metric:	P:  0.93	R:  0.94	F1:  0.93
	metric:	P: 27.18	R: 27.30	F1: 27.24
	metric:	P: 14.11	R:  5.63	F1:  8.04

Evaluation with Best
	metric:	P: 37.65	R: 37.93	F1: 37.78
	metric:	P: 10.16	R: 10.24	F1: 10.20
	metric:	P:  4.36	R:  4.41	F1:  4.38
	metric:	P:  2.48	R:  2.52	F1:  2.50
	metric:	P: 31.15	R: 31.39	F1: 31.27
	metric:	P: 16.69	R:  6.69	F1:  9.55

Evaluation with Individual
	Hypothesis #0 & Reference #0: 
		metric:	P: 35.19	R: 36.54	F1: 35.85
	Hypothesis #0 & Reference #1: 
		metric:	P: 37.04	R: 39.22	F1: 38.10
	Hypothesis #0 & Reference #2: 
		metric:	P: 33.33	R: 36.00	F1: 34.62
	Hypothesis #0 & Reference #3: 
		metric:	P: 30.56	R: 32.67	F1: 31.58
	Hypothesis #1 & Reference #0: 
		metric:	P: 25.00	R: 24.30	F1: 24.64
	Hypothesis #1 & Reference #1: 
		metric:	P: 25.96	R: 25.47	F1: 25.71
	Hypothesis #1 & Reference #2: 
		metric:	P: 22.12	R: 22.33	F1: 22[...]
{'fraction_non_alphanumeric': 0.12246826897110451, 'fraction_numerical': 0.08695652173913043, 'mean_word_length': 2.7752293577981653, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '287393', 'license': 'MIT', 'n_tokens_mistral': 3097, 'n_tokens_neox': 2584, 'n_words': 527, 'path': 'src/Evaluate_DUC.ipynb', 'repo_name': 'jamesberrypersonal/LDA_centroid_based_summarization', 'size': '7406'}