<jupyter_start><jupyter_text>Text Summarizer<jupyter_code>import nltk
nltk.download('stopwords')
from collections import Counter
from nltk.tokenize import sent_tokenize,word_tokenize
from nltk.corpus import stopwords
from string import punctuation
from heapq import nlargest
STOPWORDS = set(stopwords.words('english') + list(punctuation))
MIN_WORD_PROP, MAX_WORD_PROP = 0.1, 0.9

def compute_word_frequencies(word_sentences):
    words = [word for sentence in word_sentences 
                     for word in sentence 
                         if word not in STOPWORDS]
    counter = Counter(words)
    limit = float(max(counter.values()))
    word_frequencies = {word: freq/limit 
                                for word,freq in counter.items()}
    # Drop words if too common or too uncommon
    word_frequencies = {word: freq 
                            for word,freq in word_frequencies.items() 
                                if freq > MIN_WORD_PROP 
                                and freq < MAX_WORD_PROP}
    return word_frequencies

def sentence_score(word_sentence, word_frequencies):
    return sum([ word_frequencies.get(word,0) 
                    for word in word_sentence])

def summarize(text:str, num_sentences=3):
    """
    Summarize the text, by return the most relevant sentences
     :text the text to summarize
     :num_sentences the number of sentences to return
    """
    # Make the text lowercase
    text = text.lower()
    
    # Break text into sentences 
    sentences = sent_tokenize(text)
    
    # Break sentences into words
    word_sentences = [word_tokenize(sentence) 
                          for sentence in sentences]
    
    # Compute the word frequencies
    word_frequencies = compute_word_frequencies(word_sentences)
    
    # Calculate the scores for each of the sentences
    scores = [sentence_score(word_sentence, word_frequencies)
                     for word_sentence in word_sentences]
    sentence_scores = list(zip(sentences, scores))
    
    # Rank the sentences
    top_sentence_scores = nlargest(num_sentences, 
                                   sentence_scores,
                                   key=lambda t: t[1])
    
    # Return the top sentences
    return [t[0] for t in top_sentence_scores]
with open('data/PolarVortex.txt', 'r') as vortex_file:
    vortex_article = vortex_file.read()
vortex_article
len(sent_tokenize(vortex_article))
summarize(vortex_article)
summarize(vortex_article, num_sentences=1)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.06719681908548708, 'fraction_numerical': 0.0035785288270377734, 'mean_word_length': 2.5738636363636362, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '110066', 'license': 'MIT', 'n_tokens_mistral': 766, 'n_tokens_neox': 714, 'n_words': 193, 'path': 'Lesson6/Exercise52.ipynb', 'repo_name': 'TrainingByPackt/Introduction-to-Natural-Language-Processing', 'size': '2515'}