<jupyter_start><jupyter_text>Collocations<jupyter_code>def compute_ngrams(sequence, n):
    return list(
            zip(*(sequence[index:] 
                     for index in range(n)))
    )
compute_ngrams([1,2,3,4], 2)
compute_ngrams([1,2,3,4], 3)
def flatten_corpus(corpus):
    return ' '.join([document.strip() 
                     for document in corpus])<jupyter_output><empty_output><jupyter_text>N-grams<jupyter_code>def get_top_ngrams(corpus, ngram_val=1, limit=5):

    corpus = flatten_corpus(corpus)
    tokens = nltk.word_tokenize(corpus)

    ngrams = compute_ngrams(tokens, ngram_val)
    ngrams_freq_dist = nltk.FreqDist(ngrams)
    sorted_ngrams_fd = sorted(ngrams_freq_dist.items(), 
                              key=itemgetter(1), reverse=True)
    sorted_ngrams = sorted_ngrams_fd[0:limit]
    sorted_ngrams = [(' '.join(text), freq) 
                     for text, freq in sorted_ngrams]

    return sorted_ngrams
get_top_ngrams(corpus=norm_alice, ngram_val=2,
               limit=10)
get_top_ngrams(corpus=norm_alice, ngram_val=3,
               limit=10)<jupyter_output><empty_output><jupyter_text>N-grams and Pointwise Mututal Information<jupyter_code>from nltk.collocations import BigramCollocationFinder
from nltk.collocations import BigramAssocMeasures

finder = BigramCollocationFinder.from_documents([item.split() 
                                                for item 
                                                in norm_alice])
finder
bigram_measures = BigramAssocMeasures()                                                
finder.nbest(bigram_measures.raw_freq, 10)
finder.nbest(bigram_measures.pmi, 10)   
from nltk.collocations import TrigramCollocationFinder
from nltk.collocations import TrigramAssocMeasures

finder = TrigramCollocationFinder.from_documents([item.split() 
                                                for item 
                                                in norm_alice])
trigram_measures = TrigramAssocMeasures()                                                
finder.nbest(trigram_measures.raw_freq, 10)
finder.nbest(trigram_measures.pmi, 10)<jupyter_output><empty_output><jupyter_text>Weighted Tag-based Phrase Extraction<jupyter_code>data = open('elephants.txt', 'r+').readlines()
sentences = nltk.sent_tokenize(data[0])
len(sentences)
sentences[:3]
norm_sentences = tn.normalize_corpus(sentences, text_lower_case=False, 
                                     text_stemming=False, text_lemmatization=False, stopword_removal=False)
norm_sentences[:3]
import itertools
stopwords = nltk.corpus.stopwords.words('english')

def get_chunks(sentences, grammar = r'NP: {<DT>? <JJ>* <NN.*>+}', stopword_list=stopwords):
    
    all_chunks = []
    chunker = nltk.chunk.regexp.RegexpParser(grammar)
    
    for sentence in sentences:
        
        tagged_sents = [nltk.pos_tag(nltk.word_tokenize(sentence))]   
        
        chunks = [chunker.parse(tagged_sent) 
                      for tagged_sent in tagged_sents]
        
        wtc_sents = [nltk.chunk.tree2conlltags(chunk)
                         for chunk in chunks]    
        
        flattened_chunks = list(
                            itertools.chain.from_iterable(
                                wtc_sent for wtc_sent in wtc_sents)
                           )
        
        valid_chunks_tagged = [(status, [wtc for wtc in chunk]) 
                                   for status, chunk 
                                       in itertools.groupby(flattened_chunks, 
                                                lambda word_pos_chunk: word_pos_chunk[2] != 'O')]
        
        valid_chunks = [' '.join(word.lower() 
                                for word, tag, chunk in wtc_group 
                                    if word.lower() not in stopword_list) 
                                        for status, wtc_group in valid_chunks_tagged
                                            if status]
                                            
        all_chunks.append(valid_chunks)
    
    return all_chunks
chunks = get_chunks(norm_sentences)
chunks
from gensim import corpora, models

def get_tfidf_weighted_keyphrases(sentences, 
                                  grammar=r'NP: {<DT>? <JJ>* <NN.*>+}',
                                  top_n=10):
    
    valid_chunks = get_chunks(sentences, grammar=grammar)
                                     
    dictionary = corpora.Dictionary(valid_chunks)
    corpus = [dictionary.doc2bow(chunk) for chunk in valid_chunks]
    
    tfidf = models.TfidfModel(corpus)
    corpus_tfidf = tfidf[corpus]
    
    weighted_phrases = {dictionary.get(idx): value 
                           for doc in corpus_tfidf 
                               for idx, value in doc}
                            
    weighted_phrases = sorted(weighted_phrases.items(), 
                              key=itemgetter(1), reverse=True)
    weighted_phrases = [(term, round(wt, 3)) for term, wt in weighted_phrases]
    
    return weighted_phrases[:top_n]
get_tfidf_weighted_keyphrases(sentences=norm_sentences, top_n=30)
from gensim.summarization import keywords

key_words = keywords(data[0], ratio=1.0, scores=True, lemmatize=True)
[(item, round(score, 3)) for item, score in key_words][:25]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.085995085995086, 'fraction_numerical': 0.008694008694008694, 'mean_word_length': 1.9663677130044843, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '80968', 'license': 'Apache-2.0', 'n_tokens_mistral': 1696, 'n_tokens_neox': 1585, 'n_words': 272, 'path': 'New-Second-Edition/Ch06 - Text Summarization and Topic Models/Ch06a - Text Keyphrase Extraction.ipynb', 'repo_name': 'Sutapa-Dutta/text-analytics-with-python', 'size': '5291'}