<jupyter_start><jupyter_text>Examine word counts by genre<jupyter_code># Remove the language column
nlp_df = nlp_df.drop('language', axis=1)
# Drop songs with incorrect lyrics from the DataFrame
nlp_df = nlp_df[(nlp_df['unique_word_count']!=3878) & (nlp_df['unique_word_count']!=1153)]
nlp_df = nlp_df[(nlp_df['unique_word_count']!=1000) & (nlp_df['unique_word_count']!=880)]
nlp_df = nlp_df[(nlp_df['unique_word_count']!=842) & (nlp_df['unique_word_count']!=607)]
nlp_df = nlp_df[(nlp_df['unique_word_count']!=569) & (nlp_df['word_count']!=1383)]
len(nlp_df)
# Describe the distribution of unique word counts for each cateogry
word_distributions = {}
blues_unique_words = nlp_df[nlp_df['category']=='blues']['word_count'].describe()
word_distributions['blues'] = blues_unique_words

classical_unique_words = nlp_df[nlp_df['category']=='classical']['word_count'].describe()
word_distributions['classical'] = classical_unique_words

country_unique_words = nlp_df[nlp_df['category']=='country']['word_count'].describe()
word_distributions['country'] = country_unique_words

funk_unique_words = nlp_df[nlp_df['category']=='funk']['word_count'].describe()
word_distributions['funk'] = funk_unique_words

hiphop_unique_words = nlp_df[nlp_df['category']=='hiphop']['word_count'].describe()
word_distributions['hiphop'] = hiphop_unique_words

indie_alt_unique_words = nlp_df[nlp_df['category']=='indie_alt']['word_count'].describe()
word_distributions['indie_alt'] = indie_alt_unique_words

jazz_unique_words = nlp_df[nlp_df['category']=='jazz']['word_count'].describe()
word_distributions['jazz'] = jazz_unique_words

metal_unique_words = nlp_df[nlp_df['category']=='metal']['word_count'].describe()
word_distributions['metal'] = metal_unique_words

pop_unique_words = nlp_df[nlp_df['category']=='pop']['word_count'].describe()
word_distributions['pop'] = pop_unique_words

punk_unique_words = nlp_df[nlp_df['category']=='punk']['word_count'].describe()
word_distributions['punk'] = punk_unique_words

rnb_unique_words = nlp_df[nlp_df['category']=='rnb']['word_count'].describe()
word_distributions['rnb'] = rnb_unique_words

rock_unique_words = nlp_df[nlp_df['category']=='rock']['word_count'].describe()
word_distributions['rock'] = rock_unique_words

romance_unique_words = nlp_df[nlp_df['category']=='romance']['word_count'].describe()
word_distributions['romance'] = romance_unique_words

soul_unique_words = nlp_df[nlp_df['category']=='soul']['word_count'].describe()
word_distributions['soul'] = soul_unique_words
len(word_distributions)
# Print the distributions for each genre
word_distributions
# Create a dictionary of mean word distributions
categories = list(word_distributions.keys())
means = []
for category in word_distributions:
    means.append(word_distributions[category]['mean'])
mean_unique_word_counts = dict(zip(categories, means))
mean_unique_word_counts = dict(sorted(mean_unique_word_counts.items(), key=lambda item: item[1], reverse=True))
mean_unique_word_counts
# Find songs with the highest and lowest word counts
print('Song with the most words:')
display(nlp_df[nlp_df['word_count']==max(nlp_df['word_count'])][['song', 'artist', 'category', 'lyrics', 'word_count']])

print('Song with the least words:')
display(nlp_df[nlp_df['word_count']==min(nlp_df['word_count'])][['song', 'artist', 'category', 'lyrics', 'word_count']])

# Create DataFrame of word count distributions
ranges = []
for category in word_distributions:
    ranges.append(word_distributions[category]['max'] - word_distributions[category]['min'])

columns = ['count', 'mean', 'std', 'min', '25%', '50%', '75%', 'max', 'range']
df_rows = []
for index, category in enumerate(word_distributions):
    distribution = list(word_distributions[category])
    distribution.extend([ranges[index]])
    df_rows.append(distribution)
word_count_df = pd.DataFrame(df_rows, index=list(word_distributions.keys()), columns=columns)
word_count_df
# Save DataFrame to CSV
word_count_df.to_csv('../Data/Analysis/word_count.csv')<jupyter_output><empty_output><jupyter_text>Conclusions about word distributions by genre- The hiphop genre has the highest mean word count of 353 words- The blues genre has the lowest mean word count of 120 words- Genres with the highest average word counts:    - Hiphop: 353 words    - R&B: 232 words    - Funk: 203 words- Genres with the lowest average word counts:    - Blues: 120 words    - Jazz: 124 words    - Indie-alt: 141 words- The song with the highest word count is in the hiphop category:    - Song name: Rap God     - Artist: Eminem    - Word count: 909 words- The song with the lowest word count is in the rock category:    - Song name: Golf Hill Drive    - Artist: Boys Life    - Word count: 26 words  Examine unique word counts by genre<jupyter_code># Describe the distribution of unique word counts for each cateogry
unique_word_distributions = {}
blues_unique_words = nlp_df[nlp_df['category']=='blues']['unique_word_count'].describe()
unique_word_distributions['blues'] = blues_unique_words

classical_unique_words = nlp_df[nlp_df['category']=='classical']['unique_word_count'].describe()
unique_word_distributions['classical'] = classical_unique_words

country_unique_words = nlp_df[nlp_df['category']=='country']['unique_word_count'].describe()
unique_word_distributions['country'] = country_unique_words

funk_unique_words = nlp_df[nlp_df['category']=='funk']['unique_word_count'].describe()
unique_word_distributions['funk'] = funk_unique_words

hiphop_unique_words = nlp_df[nlp_df['category']=='hiphop']['unique_word_count'].describe()
unique_word_distributions['hiphop'] = hiphop_unique_words

indie_alt_unique_words = nlp_df[nlp_df['category']=='indie_alt']['unique_word_count'].describe()
unique_word_distributions['indie_alt'] = indie_alt_unique_words

jazz_unique_words = nlp_df[nlp_df['category']=='jazz']['unique_word_count'].describe()
unique_word_distributions['jazz'] = jazz_unique_words

metal_unique_words = nlp_df[nlp_df['category']=='metal']['unique_word_count'].describe()
unique_word_distributions['metal'] = metal_unique_words

pop_unique_words = nlp_df[nlp_df['category']=='pop']['unique_word_count'].describe()
unique_word_distributions['pop'] = pop_unique_words

punk_unique_words = nlp_df[nlp_df['category']=='punk']['unique_word_count'].describe()
unique_word_distributions['punk'] = punk_unique_words

rnb_unique_words = nlp_df[nlp_df['category']=='rnb']['unique_word_count'].describe()
unique_word_distributions['rnb'] = rnb_unique_words

rock_unique_words = nlp_df[nlp_df['category']=='rock']['unique_word_count'].describe()
unique_word_distributions['rock'] = rock_unique_words

romance_unique_words = nlp_df[nlp_df['category']=='romance']['unique_word_count'].describe()
unique_word_distributions['romance'] = romance_unique_words

soul_unique_words = nlp_df[nlp_df['category']=='soul']['unique_word_count'].describe()
unique_word_distributions['soul'] = soul_unique_words
len(unique_word_distributions)
# Print the distributions for each genre
unique_word_distributions
# Create a dictionary of mean unique word distributions
categories = list(unique_word_distributions.keys())
means = []
for category in unique_word_distributions:
    means.append(unique_word_distributions[category]['mean'])
mean_unique_word_counts = dict(zip(categories, means))
mean_unique_word_counts = dict(sorted(mean_unique_word_counts.items(), key=lambda item: item[1], reverse=True))
mean_unique_word_counts
# Find songs with the highest and lowest unique word counts
print('Song with the most unique words:')
display(nlp_df[nlp_df['unique_word_count']==max(nlp_df['unique_word_count'])][['song', 'artist', 'category', 'lyrics', 'unique_word_count']])

print('Genres with songs that have the minimum unique word count of 25 words:')
minimum_genres = list(nlp_df[nlp_df['unique_word_count']==min(nlp_df['unique_word_count'])]['category'])
print(list(set(minimum_genres)), '\n')

min_word_count_df = nlp_df[nlp_df['unique_word_count']==min(nlp_df['unique_word_count'])]
print(f'{len(min_word_count_df)} songs have the minimum unique word count of 25 words.')

for genre in list(set(minimum_genres)):
    print(genre, minimum_genres.count(genre))
# Create DataFrame of unique word count distributions
ranges = []
for category in unique_word_distributions:
    ranges.append(unique_word_distributions[category]['max'] - unique_word_distributions[category]['min'])

columns = ['count', 'mean', 'std', 'min', '25%', '50%', '75%', 'max', 'range']
df_rows = []
for index, category in enumerate(unique_word_distributions):
    distribution = list(unique_word_distributions[category])
    distribution.extend([ranges[index]])
    df_rows.append(distribution)
unique_word_count_df = pd.DataFrame(df_rows, index=list(unique_word_distributions.keys()), columns=columns)
unique_word_count_df
# Save DataFrame to CSV
unique_word_count_df.to_csv('../Data/Analysis/unique_word_count.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14279319606087734, 'fraction_numerical': 0.008504923903312443, 'mean_word_length': 9.65196662693683, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '664656', 'license': 'MIT', 'n_tokens_mistral': 3072, 'n_tokens_neox': 2923, 'n_words': 520, 'path': 'Analysis/Notebooks/genre_by_word_count.ipynb', 'repo_name': 'hannahkidwell/NLP-Spotify-Genres', 'size': '8936'}