<jupyter_start><jupyter_text>Instructions:1. Compute the most important key-words (a key-word can be between 1-3 words)2. Choose the top n words from the previously generated list. Compare these key- words with all the words occurring in all of the transcripts.3. Generate a score (rank) for these top n words based on analysed transcripts.-----------------------------------------------------------------------------<jupyter_code>from sklearn.feature_extraction import stop_words
from nltk.corpus import stopwords
import math
from textblob import TextBlob as tb
with open("scripts/script.txt", "r") as f:
    data = f.read()

#with open("scripts/script.txt", "r") as f:
#    data2 = f.readlines()

#for line in data:
#    words = data.split()
with open("scripts/transcript_1.txt", "r") as t1:
    t1 = t1.read()
    
with open("scripts/transcript_2.txt", "r") as t2:
    t2 = t2.read()

with open("scripts/transcript_3.txt", "r") as t3:
    t3 = t3.read()<jupyter_output><empty_output><jupyter_text>PRE-PROCESSING!<jupyter_code>from spacy.en import English
import nltk
from sklearn.feature_extraction.stop_words import ENGLISH_STOP_WORDS

parser = English()
parsedData = parser(data)
# All you have to do is iterate through the parsedData
# Each token is an object with lots of different properties
# A property with an underscore at the end returns the string representation
# while a property without the underscore returns an index (int) into spaCy's vocabulary
# The probability estimate is based on counts from a 3 billion word
# corpus, smoothed using the Simple Good-Turing method.
for i, token in enumerate(parsedData[0:2]):
    print("original:", token.orth, token.orth_)
    print("lowercased:", token.lower, token.lower_)
    print("lemma:", token.lemma, token.lemma_)
    print("shape:", token.shape, token.shape_)
    print("prefix:", token.prefix, token.prefix_)
    print("suffix:", token.suffix, token.suffix_)
    print("log probability:", token.prob)
    print("Brown cluster id:", token.cluster)
    print("----------------------------------------")<jupyter_output>original: 6832 Food
lowercased: 960 food
lemma: 960 food
shape: 354724 Xxxx
prefix: 3792 F
suffix: 155057 ood
log probability: -11.701742172241211
Brown cluster id: 838
----------------------------------------
original: 474 is
lowercased: 474 is
lemma: 488 be
shape: 21581 xx
prefix: 570 i
suffix: 474 is
log probability: -4.457748889923096
Brown cluster id: 762
----------------------------------------<jupyter_text>TF-IDF<jupyter_code>from sklearn.metrics import accuracy_score
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer

def tf(word, blob):
    return blob.words.count(word) / len(blob.words)

def n_containing(word, bloblist):
    return sum(1 for blob in bloblist if word in blob.words)

def idf(word, bloblist):
    return math.log(len(bloblist) / (1 + n_containing(word, bloblist)))

def tfidf(word, blob, bloblist):
    return tf(word, blob) * idf(word, bloblist)

bloblist = []
[bloblist.append(tb(doc)) for doc in [data, t1, t2, t3]]

for i, blob in enumerate(bloblist):
    print("Top words in document {}".format(i + 1))
    scores = {word: tfidf(word, blob, bloblist) for word in blob.words}
    sorted_words = sorted(scores.items(), key=lambda x: x[1], reverse=True)
    for word, score in sorted_words[:3]:
        print("Word: {}, TF-IDF: {}".format(word, round(score, 5)))
CountVectorizer(data)
tf = TfidfVectorizer(analyzer='word', ngram_range=(1,3), min_df = 0, stop_words = 'english')

tfidf_matrix = tf.fit_transform(data2)
feature_names = tf.get_feature_names() 
tfidf_matrix.shape, len(feature_names)
dense = tfidf_matrix.todense()
episode = dense[0].tolist()[0]
phrase_scores = [pair for pair in zip(range(0, len(episode)), episode) if pair[1] > 0]
sorted_phrase_scores = sorted(phrase_scores, key=lambda t: t[1] * -1)
for phrase, score in [(feature_names[word_id], score) for (word_id, score) in sorted_phrase_scores][:20]:
   print('{0: <20} {1}'.format(phrase, score))
def freq(word, tokens):
    return tokens.count(word)


#Compute the frequency for each term.
vocabulary = []
docs = {}
all_tips = []
for tip in (venue.tips()):
    tokens = tokenizer.tokenize(tip.text)

    bi_tokens = bigrams(tokens)
    tri_tokens = trigrams(tokens)
    tokens = [token.lower() for token in tokens if len(token) > 2]
    tokens = [token for token in tokens if token not in stopwords]

    bi_tokens = [' '.join(token).lower() for token in bi_tokens]
    bi_tokens = [token for token in bi_tokens if token not in stopwords]

    tri_tokens = [' '.join(token).lower() for token in tri_tokens]
    tri_tokens = [token for token in tri_tokens if token not in stopwords]

    final_tokens = []
    final_tokens.extend(tokens)
    final_tokens.extend(bi_tokens)
    final_tokens.extend(tri_tokens)
    docs[tip.text] = {'freq': {}}

    for token in final_tokens:
        docs[tip.text]['freq'][token] = freq(token, final_tokens)

print docs<jupyter_output><empty_output><jupyter_text>RAKE<jupyter_code>from rake_nltk import Rake

r = Rake() # Uses stopwords for english from NLTK, and all puntuation characters.
# If you want to provide your own set of stop words and punctuations to
# r = Rake(<list of stopwords>, <string of puntuations to ignore>

r.extract_keywords_from_text(data)
r.get_ranked_phrases_with_scores() # To get keyword phrases ranked highest to lowest.<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14822316332167187, 'fraction_numerical': 0.02448904437488492, 'mean_word_length': 3.924750679963735, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '420667', 'license': 'MIT', 'n_tokens_mistral': 1850, 'n_tokens_neox': 1731, 'n_words': 544, 'path': 'notebooks/sketch.ipynb', 'repo_name': 'BelalC/keyword_i2x', 'size': '5431'}