<jupyter_start><jupyter_text>EDA<jupyter_code>import pandas as pd
from langdetect import detect_langs
import matplotlib.pyplot as plt
import seaborn as sns
raw_data_dir= '../data/wrangled/'
raw_data_name ='wrangled_data.csv'
df = pd.read_csv(raw_data_dir+raw_data_name,index_col=0)
# Number of authors
df.author.nunique()
# Statistics of how many poems from different authores there are
df.author.value_counts()[(df.author.value_counts()>1)].describe() 
authors =  df.author.value_counts()[(df.author.value_counts()>1)]
author_list= ['Shakespeare',
 '<NAME>',
 '<NAME>',
 '<NAME>',
 '<NAME>',
 '<NAME>',
 '<NAME>',
 '<NAME>',
 '<NAME>',
 '<NAME>',
 '<NAME>',
 '<NAME>',
 '<NAME>',
 '<NAME>',
 '<NAME>',
 '<NAME>',
 '<NAME>',
 '<NAME>',
 '<NAME>',
 '<NAME>',
 '<NAME>',
 '<NAME>',
 '<NAME>',
 '<NAME>']

author_list= ['Shakespeare',
 'Tennyson',
 'Wordsworth',
 'Donne',
 'Sidney',
 'Ashbery',
 'Swinburne',
 'Young',
 'Browning',
 'Merwin',
 'Hardy',
 'Spenser',
 'Komunyakaa',
 'Yeats',
 'Whitman',
 'Longfellow',
 'Shelley',
 'Collins',
 'Masters',
 'Notley',
 'Keats',
 'Arnold',
 'Barrett',
 'Herbert']
data = authors.values[:24]
ser = pd.Series(data, index =author_list ) 
# Bar plot of poems per authors
FONTSIZE= 14
cols =ser[:15]
fig = plt.figure(figsize=(15,12))
ax = sns.barplot(cols.index, cols.values);
#ax.set_title('Poetry scrap from the poetry foundation', fontsize=25)
plt.ylabel('Poems per author', fontsize=35)
plt.xticks( rotation='vertical', fontsize=25);
plt.yticks( fontsize=28);
plt.show()
fig.savefig("../media/authors.png")
# Bar plot of poems per authors
FONTSIZE= 14
cols =authors[:100]
fig = plt.figure(figsize=(15,12))
ax = sns.barplot(cols.index, cols.values);
#ax.set_title('Poetry scrap from the poetry foundation', fontsize=25)
plt.ylabel('Poems per author', fontsize=20)
plt.xticks( rotation='vertical', fontsize=10);
plt.yticks( fontsize=15);
plt.show()
#fig.savefig("../media/authors.png")
#histogram of number of poems per authors
cols.hist(bins=45)<jupyter_output><empty_output><jupyter_text>**Insights:** - At least 80 authors have written have written more than 20 poems.<jupyter_code># Words lenght per poem cut above WORD_LIMIT
WORD_LIMIT= 2000
df.length_in_words[df.length_in_words <WORD_LIMIT].hist(bins=45)
# Word lenght per poem cut below WORD_LIMIT
df.length_in_words[df.length_in_words >=WORD_LIMIT].hist(bins=45)
# Length of poems
fig = plt.figure(1, figsize=(9, 6))
lengths= df.length_in_words
plt.boxplot([lengths[lengths<1000]])
#fig.savefig('../plots_tables/catplots.png')<jupyter_output><empty_output><jupyter_text>**Poem lenghts insight**Most poems are shorter than a few hundred words. But some are as 20_000 words. Hence, it seems like we need to transform those poems into shorter versions.<jupyter_code>from wordcloud import WordCloud
from nltk.corpus import stopwords
stopwords_en = stopwords.words('english')

def cloud_maker(text, background = 'white', withStopwords=True):
    
    if withStopwords == True: _stw = stopwords_en
    else: _stw = []
    wordcloud = WordCloud(background_color = background, 
                stopwords = _stw)
    wordcloud.generate(text)
    return wordcloud
txt_label = df.content.values
txt_label = ' '.join(txt_label)
cloud = cloud_maker(txt_label, background = 'white')
fig = plt.figure(1, figsize=(15, 12))
plt.imshow(cloud.recolor(random_state=1234), 
              interpolation = 'bilinear')

#    plt.gca().set_title(_labels[i],fontsize=15)
#    plt.axis("off")
#fig.savefig('../plots_tables/clouds.png')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1656771275091886, 'fraction_numerical': 0.021204410517387615, 'mean_word_length': 5.700757575757576, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '555227', 'license': 'MIT', 'n_tokens_mistral': 1342, 'n_tokens_neox': 1284, 'n_words': 288, 'path': 'wrangling_eda/eda.ipynb', 'repo_name': 'reneang17/authorencoder', 'size': '3632'}