<jupyter_start><jupyter_text>Exercise 1Implement the LSI using SVD. Use the the documents given below assamples.__Note__: Use Numpy's linalg package to understand the internals of SVD1. information extraction systems2. natural language processing3. speech signal systems4. speech processingUse "speech systems" as the query to find out whether the results matchthe answer for Question 5 (Assignment 2).<jupyter_code>from gensim import corpora, similarities
from gensim import models
# Read (construct) the Corpus
corpus = [
    "information extraction systems",
    "natural language processing",
    "speech signal systems",
    "speech processing"
]
# Tokenize
corpus_text = [doc.split() for doc in corpus]
corpus_text
# Reference: https://radimrehurek.com/gensim/tut2.html
# Create Coprus Dictionary
dictionary = corpora.Dictionary(corpus_text)
# Create Document-Term Matrix (Bag-of-Words)
doc_term_matrix = [dictionary.doc2bow(tokens) for tokens in corpus_text]
doc_term_matrix
# Create TF-IDF Matrix - by fit(ting) the model
tfidf = models.TfidfModel(doc_term_matrix)
# Transform (Apply) the model
corpus_tfidf = tfidf[doc_term_matrix]
# Create LSI Transformation
lsi = models.LsiModel(corpus_tfidf, id2word=dictionary)
lsi.print_topics()
lsi.show_topics()
# Create Query Document
query_doc = "speech systems"
query_vec_bow = dictionary.doc2bow(query_doc.split())
# convert the query to LSI space
query_vec_lsi = lsi[query_vec_bow]
query_vec_lsi
# transform corpus to LSI space and index it
# Reference: https://radimrehurek.com/gensim/tut3.html
index = similarities.MatrixSimilarity(lsi[corpus_tfidf])
# Obtain similarity against all the documents in the corpus
sims = index[query_vec_lsi]
print(list(enumerate(sims)))<jupyter_output>[(0, 0.33579946), (1, 0.0), (2, 0.82253736), (3, 0.7123382)]<jupyter_text>Exercise 2Try to improve the incomplete code (shown during the demo 2)(Topicmodeling using BBC corpus) from the githubOnce completed, post your experience in the discussion forum for the benefit of others<jupyter_code>import os
import pathlib
import gensim
from gensim.models import LsiModel
from gensim import models
from gensim import corpora
from gensim.utils import lemmatize
import nltk
from nltk.stem import PorterStemmer
from nltk.corpus import stopwords
from nltk.stem.snowball import EnglishStemmer # Yuva
from nltk.stem import WordNetLemmatizer # Yuva
from gensim.parsing.preprocessing import remove_stopwords, stem_text
from gensim.parsing.preprocessing import strip_numeric, strip_short,strip_multiple_whitespaces,strip_non_alphanum,strip_punctuation,strip_tags,preprocess_string
import pandas as pd
from gensim import similarities
from nltk.tokenize import RegexpTokenizer
from nltk.corpus import stopwords
from nltk.stem.porter import PorterStemmer
from gensim.models.coherencemodel import CoherenceModel
import matplotlib.pyplot as plt
from pprint import pprint
#read the data
corpus_file_name = pathlib.Path("bbc-text.csv")

if corpus_file_name.exists():
    corpus_dir = corpus_file_name
else:
    corpus_dir = 'https://raw.githubusercontent.com/Ramaseshanr/anlp/master/corpus/bbc-text.csv'
    
df_corpus = pd.read_csv(corpus_dir,names=['category', 'text'])

if not corpus_file_name.exists():
    df_corpus.to_csv(corpus_file_name,index=False,header=False)
corpus = df_corpus['text'].values.tolist()
corpus = corpus[1:]
my_filter = [
    lambda x: x.lower(), strip_tags, strip_punctuation,
    strip_multiple_whitespaces, strip_numeric,
    remove_stopwords, strip_short, stem_text
]

def preprocessing(corpus):
    stemmer = EnglishStemmer()
    lemmatizer = WordNetLemmatizer()
    stop_words = stopwords.words('english')
    for document in corpus:
        doc = strip_numeric(document)
        doc = strip_multiple_whitespaces(doc)
        doc = remove_stopwords(doc)
        strip_tags(doc)
        doc = strip_short(doc,3)
        #doc = stem_text(doc)
        doc = strip_punctuation(doc)
        tokens = gensim.utils.tokenize(doc, lower=True)
        tokens = [token for token in tokens if token not in stop_words ]
        #tokens = [stemmer.stem(token) for token in tokens]
        tokens = [lemmatizer.lemmatize(token) for token in tokens]
        yield tokens
        #yield gensim.utils.tokenize(doc, lower=True)

def preprocessing2(corpus):
    for document in corpus:
        doc = strip_numeric(document)
        doc = strip_multiple_whitespaces(doc)
        doc = remove_stopwords(doc)
        strip_tags(doc)
        doc = strip_short(doc,3)
        #doc = stem_text(doc)
        doc = strip_punctuation(doc)
        yield gensim.utils.tokenize(doc, lower=True)

#texts = preprocessing(corpus)
#for text in texts:
#    print(list(text))

texts = preprocessing(corpus)
dictionary = corpora.Dictionary(texts)
dictionary.save_as_text("corpus_dict.txt")
corpus_vocab = dictionary.itervalues()
print("Corpus Vocab Size: ", len(corpus_vocab), ", Corpus Dictionary Size: ", len(dictionary.token2id))

with open('corpus_vocab.txt', 'w') as f:
    v = sorted(corpus_vocab, key=len)
    for w in v:
        f.write(w + '\n')
dictionary.filter_extremes(no_below=1, keep_n=12000)
print('After filtering: ', len(dictionary.itervalues()))
dictionary.save_as_text("corpus_dict2.txt")
doc_term_matrix = [dictionary.doc2bow(tokens) for tokens in preprocessing(corpus)]
tfidf = models.TfidfModel(doc_term_matrix)
corpus_tfidf = tfidf[doc_term_matrix]
lsi = models.LsiModel(corpus_tfidf, id2word=dictionary)  # initialize an LSI transformation
lsi.projection.k
lsi.projection.s
pprint(lsi.print_topics(num_topics=5, num_words=25))<jupyter_output>[(0,
  '-0.121*"labour" + -0.117*"election" + -0.102*"blair" + -0.101*"film" + '
  '-0.100*"game" + -0.099*"tax" + -0.098*"brown" + -0.093*"party" + '
  '-0.092*"government" + -0.088*"people" + -0.086*"tory" + -0.084*"bn" + '
  '-0.082*"mobile" + -0.075*"minister" + -0.075*"economy" + -0.072*"music" + '
  '-0.072*"service" + -0.068*"phone" + -0.067*"new" + -0.066*"best" + '
  '-0.065*"sale" + -0.064*"market" + -0.064*"howard" + -0.064*"growth" + '
  '-0.064*"rate"'),
 (1,
  '-0.265*"labour" + -0.252*"election" + -0.215*"blair" + -0.201*"brown" + '
  '-0.192*"tax" + -0.188*"tory" + -0.186*"party" + 0.176*"film" + 0.134*"game" '
  '+ -0.128*"chancellor" + 0.122*"award" + -0.121*"howard" + 0.117*"mobile" + '
  '-0.111*"minister" + -0.106*"prime" + -0.105*"government" + 0.099*"music" + '
  '0.097*"best" + 0.088*"phone" + -0.085*"lib" + -0.085*"conservative" + '
  '0.081*"player" + -0.079*"kennedy" + -0.071*"budget" + -0.070*"leader"'),
 (2,
  '-0.200*"mobile" + 0.187*"film" + -0.153*"phone[...]
{'fraction_non_alphanumeric': 0.12957618107245936, 'fraction_numerical': 0.04222998632842169, 'mean_word_length': 4.735191637630662, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '638356', 'license': 'MIT', 'n_tokens_mistral': 2537, 'n_tokens_neox': 2334, 'n_words': 516, 'path': 'Week2-Assignment.ipynb', 'repo_name': 'gymk/ANLP', 'size': '6583'}