<jupyter_start><jupyter_text>Load up the songs<jupyter_code>songs = corpus.load_songs(corpus.processed_data_dirs()[0])<jupyter_output><empty_output><jupyter_text>Add rank differential<jupyter_code>for song in songs:
    song['rank_differential'] = (int(song['rank_this_week']) - int(song['rank_last_week'])) / 100<jupyter_output><empty_output><jupyter_text>NLTK default english stop words<jupyter_code>from nltk.corpus import stopwords
print(stopwords.words('english'))<jupyter_output>['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', 'her', 'hers', 'herself', 'it', 'its', 'itself', 'they', 'them', 'their', 'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', 'these', 'those', 'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does', 'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of', 'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after', 'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very', 's', 't', 'can', 'will', 'just', 'don', 'should', 'no[...]<jupyter_text>Custom stop words list<jupyter_code>print(corpus.stop_words())<jupyter_output>['a', 'about', 'above', 'across', 'after', 'again', 'against', 'all', 'almost', 'alone', 'along', 'already', 'also', 'although', 'always', 'among', 'an', 'and', 'another', 'any', 'anybody', 'anyone', 'anything', 'anywhere', 'are', 'area', 'areas', 'around', 'as', 'ask', 'asked', 'asking', 'asks', 'at', 'away', 'b', 'back', 'backed', 'backing', 'backs', 'be', 'became', 'because', 'become', 'becomes', 'been', 'before', 'began', 'behind', 'being', 'beings', 'best', 'better', 'between', 'big', 'both', 'but', 'by', 'c', 'came', 'can', 'cannot', 'case', 'cases', 'certain', 'certainly', 'clear', 'clearly', 'come', 'could', 'd', 'did', 'differ', 'different', 'differently', 'do', 'does', 'done', 'down', 'down', 'downed', 'downing', 'downs', 'during', 'e', 'each', 'early', 'either', 'end', 'ended', 'ending', 'ends', 'enough', 'even', 'evenly', 'ever', 'every', 'everybody', 'everyone', 'everything', 'everywhere', 'f', 'face', 'faces', 'fact', 'facts', 'far', 'felt', 'few', 'find', 'finds', 'first[...]<jupyter_text>Get normalized frequency of wordsLook at all distributions of words in a song individually and normalize by the number of songs. This is a better approach than looking at each word as a total bag of words in all songs. Some of these songs have super repitive words in one song which throws off the overall distribution.<jupyter_code>import string
import unicodedata
import sys

from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from nltk import FreqDist

import pandas as pd

punc_tbl = dict.fromkeys(i for i in range(sys.maxunicode)
                      if unicodedata.category(chr(i)).startswith('P'))

def remove_punctuation(text):
    return text.translate(punc_tbl)

def remove_stop_words(words):
    """Remove stop words from a list of words."""
    wl = []    
    stop_words = corpus.stop_words()
    for word in words:
        word = word.lower()
        if word not in stop_words:
            wl.append(word)
    
    return wl

normalized_word_frequencies = {}

for song in songs:
    song['word_lyrics'] = remove_stop_words(word_tokenize(remove_punctuation(song['lyrics'])))    
    dist = FreqDist(song['word_lyrics'])

    for w in dist:
        if not w in normalized_word_frequencies:
            normalized_word_frequencies[w] = 0

        normalized_word_frequencies[w] = normalized_word_frequencies[w] + dist.freq(w)
        
for w, v in normalized_word_frequencies.items():
    normalized_word_frequencies[w] = v / len(songs)<jupyter_output><empty_output><jupyter_text>Plot the word distributions<jupyter_code>df = pd.DataFrame.from_dict(normalized_word_frequencies, orient='index')
word_freq = df.nlargest(25, 0).plot(kind='bar', title='Normalized Word Frequency', legend=False)
word_freq.set_xlabel("Word")
word_freq.set_ylabel("Distribution")<jupyter_output><empty_output><jupyter_text>Normalize and stem words<jupyter_code>from nltk.stem import PorterStemmer

def stem_words(words):
    stemmer = PorterStemmer()
    return [stemmer.stem(w) for w in words]

normalized_word_frequencies = {}

for song in songs:
    song['word_lyrics'] = stem_words(remove_stop_words(word_tokenize(remove_punctuation(song['lyrics']))))
    dist = FreqDist(song['word_lyrics'])

    for w in dist:
        if not w in normalized_word_frequencies:
            normalized_word_frequencies[w] = 0

        normalized_word_frequencies[w] = normalized_word_frequencies[w] + dist.freq(w)
        
for w, v in normalized_word_frequencies.items():
    normalized_word_frequencies[w] = v / len(songs)
df = pd.DataFrame.from_dict(normalized_word_frequencies, orient='index')
word_freq = df.nlargest(25, 0).plot(kind='bar')
word_freq.set_xlabel("Word")
word_freq.set_ylabel("Distribution")<jupyter_output><empty_output><jupyter_text>Stemmed and normalized swear word frequency<jupyter_code>swear_words = set(stem_words(corpus.swear_words()))

normalized_word_frequencies = {}

for song in songs:
    song['word_lyrics'] = stem_words(remove_stop_words(word_tokenize(remove_punctuation(song['lyrics']))))
    dist = FreqDist(song['word_lyrics'])

    for sw in swear_words:
        if not sw in normalized_word_frequencies:
            normalized_word_frequencies[sw] = 0

        normalized_word_frequencies[sw] = normalized_word_frequencies[sw] + dist.freq(sw)
        
for w, v in normalized_word_frequencies.items():
    normalized_word_frequencies[w] = v / len(songs)
df = pd.DataFrame.from_dict(normalized_word_frequencies, orient='index')
word_freq = df.nlargest(10, 0).plot(kind='bar', title='Swear Word Frequency', legend=False)
word_freq.set_xlabel("Word")
word_freq.set_ylabel("Distribution")<jupyter_output><empty_output><jupyter_text>Sentiment analysis with Vader<jupyter_code>from nltk.sentiment.vader import SentimentIntensityAnalyzer
from nltk.tokenize import sent_tokenize


def split_sentences(text):
    sentences = []
    for sentence in text.split('\n'):
        sentence = sentence.strip()
        
        if sentence:
            sentences.append(sentence)

    return sentences


def sentiment_for_song(text):
    sid = SentimentIntensityAnalyzer()

    # can't use sent_tokenize with lyrics... just split on newline
    sentences = split_sentences(text)
    total_ss = {
        'negative': 0,
        'positive': 0,
        'neutral': 0,
        'compound': 0
    }

    for sentence in sentences:
        ss = sid.polarity_scores(sentence)
        total_ss['negative'] = total_ss['negative'] + ss['neg']
        total_ss['positive'] = total_ss['positive'] + ss['pos']
        total_ss['neutral'] = total_ss['neutral'] + ss['neu']
        total_ss['compound'] = total_ss['compound'] + ss['compound']
        
    for key in total_ss:
        total_ss[key] = total_ss[key] / len(sentences)
        
    return total_ss

all_sentiments = {}
for song in songs:
    sentiments = sentiment_for_song(song['lyrics'])

    # add sentiment to song data
    song['positive_sentiment'] = sentiments['positive']
    song['negative_sentiment'] = sentiments['negative']
    song['neutral_sentiment'] = sentiments['neutral']
    
    for key in sentiments:
        if key not in all_sentiments:
            all_sentiments[key] = 0
            
        all_sentiments[key] = all_sentiments[key] + sentiments[key]

# normalize for number of songs
for key in all_sentiments:
    all_sentiments[key] = all_sentiments[key] / len(songs)

print(all_sentiments)<jupyter_output>{'negative': 0.08538988774533011, 'positive': 0.11905898567659678, 'neutral': 0.7955515403012989, 'compound': 0.03935044543716692}<jupyter_text>Plot sentiments<jupyter_code>all_sentiments.pop('compound', 0)
df = pd.DataFrame.from_dict(all_sentiments, orient='index')
sent_plot = df.nlargest(10, 0).plot(kind='bar', title='Sentiment Distribution', legend=False)
sent_plot.set_xlabel("Sentiment")
sent_plot.set_ylabel("Distribution")<jupyter_output><empty_output><jupyter_text>5 most negative songs<jupyter_code>songs_df = pd.DataFrame.from_dict(songs)
songs_df.nlargest(5, 'negative_sentiment')<jupyter_output><empty_output><jupyter_text>5 most positive songs<jupyter_code>songs_df.nlargest(5, 'positive_sentiment')<jupyter_output><empty_output><jupyter_text>5 most neutral songs<jupyter_code>songs_df.nlargest(5, 'neutral_sentiment')<jupyter_output><empty_output><jupyter_text>How repititive?Check how repititive a song is by looking at the number of occurences each phrase or sentence has. A lot of songs are repititive due to the nature of a chorus etc.. however some might be super repititve.<jupyter_code>def song_repitiveness(lyrics, rate=2):
    # split song on sentence and find unique sentences
    sentences = split_sentences(lyrics)
    unique_sentences = set(sentences)
    
    total_sentences = len(sentences)
    total_unique_sentences = len(unique_sentences)
    
    # collect frequency of unique sentences and calculate reptitiveness
    repititive_rate = 0
    frequency = 0
    for usentence in unique_sentences:
        for sentence in sentences:
            if usentence == sentence:
                frequency = frequency + 1

        # only calc. reptitiveness rate if frequency rate cutoff is met
        if frequency >= rate:
            repititive_rate = repititive_rate + (frequency / total_sentences)
            
        frequency = 0
        
    return repititive_rate

for song in songs:
    song['repititiveness'] = song_repitiveness(song['lyrics'], rate=2)
    
songs_df = pd.DataFrame.from_dict(songs)
plot = songs_df.sort_values('rank_this_week').plot(x='title', y=('repititiveness'), kind='bar', title='Song Repetitiveness\nOrdered by Current Rank', stacked=True, figsize=(20,5))
mean = songs_df.mean()['repititiveness']
median = songs_df.median()['repititiveness']
mean_line = plot.axhline(mean, color='r', label='Mean', linestyle='--')
median_line = plot.axhline(median, color='g', label='Median', linestyle='--')
plot.legend(handles=[mean_line, median_line], loc=1)
songs_df.describe()
songs_df.sort_values('repititiveness').tail(10)<jupyter_output><empty_output><jupyter_text>Data correlations<jupyter_code>songs_df.corr()<jupyter_output><empty_output><jupyter_text>Repititiveness and negative sentiment<jupyter_code>songs_df[['rank_this_week','rank_last_week']] = songs_df[['rank_this_week','rank_last_week']].apply(pd.to_numeric)
songs_df.plot('repititiveness', 'negative_sentiment', kind='scatter', title='Negative Sentiment and Repetitiveness')<jupyter_output><empty_output><jupyter_text>Repititiveness and positive sentiment<jupyter_code>songs_df.plot('repititiveness', 'positive_sentiment', kind='scatter', title='Positive Sentiment and Repetitiveness')<jupyter_output><empty_output><jupyter_text>Rank and positive sentiment<jupyter_code>songs_df.plot('rank_this_week', 'positive_sentiment', kind='scatter', title='Positive Sentiment and Current Rank')<jupyter_output><empty_output><jupyter_text>Rank and negative sentiment<jupyter_code>songs_df.plot('rank_this_week', 'negative_sentiment', kind='scatter', title='Negative Sentiment and Current Rank')<jupyter_output><empty_output><jupyter_text>Rank and repititiveness<jupyter_code>songs_df.plot('rank_this_week', 'repititiveness', kind='scatter', title='Repetitiveness and Current Rank')<jupyter_output><empty_output><jupyter_text>Common bigrams - bag of words - not normalized<jupyter_code>from nltk.util import ngrams

all_words = []
for song in songs:
    all_words = all_words + song['word_lyrics']

ngrams = ngrams(all_words, 2)
dist = FreqDist(ngrams)
dist.plot(20, title='Common BiGrams')<jupyter_output><empty_output><jupyter_text>Common trigrams - bag of words - not normalized<jupyter_code>from nltk.util import ngrams

all_words = []
for song in songs:
    all_words = all_words + song['word_lyrics']

ngrams = ngrams(all_words, 3)
dist = FreqDist(ngrams)
dist.plot(20, title='Common TriGrams')<jupyter_output><empty_output><jupyter_text>Normalized trigrams<jupyter_code>from nltk.util import ngrams

normalized_ngram_frequencies = {}

for song in songs:
    ngs = list(ngrams(song['word_lyrics'], 3))
    unique_ngs = set(ngs)
    dist = FreqDist(ngs)

    for ngram in unique_ngs:
        if not ngram in normalized_ngram_frequencies:
            normalized_ngram_frequencies[ngram] = 0
            
        normalized_ngram_frequencies[ngram] = normalized_ngram_frequencies[ngram] + dist.freq(ngram)
        
for ngram, v in normalized_ngram_frequencies.items():
    normalized_ngram_frequencies[ngram] = v / len(songs)
df = pd.DataFrame.from_dict(normalized_ngram_frequencies, orient='index')
word_freq = df.nlargest(10, 0).plot(kind='bar')
word_freq.set_xlabel("NGram")
word_freq.set_ylabel("Distribution")
songs_df.plot(x='positive_sentiment', y='rank_differential', kind='scatter')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14836772983114446, 'fraction_numerical': 0.009380863039399626, 'mean_word_length': 5.597029702970297, 'pattern_counts': {'":': 0, '<': 90, '<?xml version=': 0, '>': 91, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '23', 'dir': 'jupyter-structured-clean-dedup', 'id': '388837', 'license': 'MIT', 'n_tokens_mistral': 4781, 'n_tokens_neox': 4522, 'n_words': 972, 'path': 'notebooks/exploratory/04-clean-lyric-analysis.ipynb', 'repo_name': 'tylerwmarrs/billboard-hot-100-lyric-analysis', 'size': '13325'}