<jupyter_start><jupyter_text>Story Data Exploration<jupyter_code>#Import required libraries
import matplotlib.pyplot as plt 
import pandas as pd  
import numpy as np
import seaborn as  sns
#Load the merged file
storycontentdata = pd.read_csv("stories_content.csv")
#View structure of the file
storycontentdata.info()
#View summary statistics for the numeric variables. Variable of interest is #reads.  
storycontentdata.describe()
#Observations: Average number of reads is 119 for stories.#Max is 27K, Min is 0 reads.
#Only data of published stories is present in the corpus <stories_status=1>
#View initial rows from the file to get an idea of the content
storycontentdata.head()
#Observations: Stories are in multiple languages. Story category information seems to be missing. 
#View the number of stories published in various languages
#<For a count, uncomment below line of code.>
#storycontentdata.language_name.value_counts()
#<For graphical view, use below lines of code.>
plt.figure(figsize = (30,10))

sns.set(style="darkgrid")
# Plot
sns.countplot(x='language_name', data=storycontentdata).set_title('Language')

# Setting labels
plt.xticks(rotation = 45)
plt.show()
#Observation: Most of the stories are in English language and Hindi so would be a good place to start
#analysis and tag generation as we would have sufficient corpus. 
#View the %distribution of stories created by children v/s adults.
plt.figure(figsize = (5,4))

temp = pd.DataFrame(storycontentdata.is_child_created_story.value_counts()/sum(storycontentdata.is_child_created_story.value_counts())*100)
fig = sns.barplot(temp.index,temp.is_child_created_story)
fig.set_title('Children Stories (Percentage)')
fig.set_xlabel('Is child created story')
fig.set_ylabel('Percentage')
#Setting y limits
plt.ylim([0,100])
#plt.xlabel='Is child created story'
#plt.ylabel='Percentage'

plt.show()

#Observation: A very few stories are child created. While on the portal, there are many 
#child created stories. Needs to be checked if we are missing these.
#View the % distribution of stories which have been recommended. 

plt.figure(figsize = (5,4))

temp = pd.DataFrame(storycontentdata.is_recommended_story.value_counts()/sum(storycontentdata.is_recommended_story.value_counts())*100)
fig = sns.barplot(temp.index,temp.is_recommended_story)
fig.set_title('Recommended Stories (Percentage)')
fig.set_xlabel('Is recommended story')
fig.set_ylabel('Percentage')
#Setting y limits
plt.ylim([0,100])
plt.show()
#Observation: There are a significant number of stories which are recommended to others for reading. 

# To get numeric count in place of % distribution, uncomment below lines of code
# plt.figure(figsize = (5,10))

# sns.set(style="darkgrid")
# # Plot
# sns.countplot(x='is_recommended_story', data=storycontentdata).set_title('Recommended Stories')
# # Setting labels
# plt.show()
#View the story derivation type distribution to see what % of stories are original v/s translated.
plt.figure(figsize = (5,4))

temp = pd.DataFrame(storycontentdata.story_derivation_type.value_counts()/sum(storycontentdata.story_derivation_type.value_counts())*100)
fig = sns.barplot(temp.index,temp.story_derivation_type)
fig.set_title('Stories Derivation Type(Percentage)')
fig.set_xlabel('Story Derivation Type')
fig.set_ylabel('Percentage')

#Setting y limits
plt.ylim([0,100])
plt.show()
#Observation: A large number of stories are translated. It may be worthwhile exploring how tagging 
#original v/s translated stories works. Are tags of translated stories similar to those in original?
#Perhaps we can generate tags for original stories and then use them for translated ones in 
#respective languages? 
#View distribution of organization name to check which other publishers are involved
plt.figure(figsize = (10,10))

temp = pd.DataFrame(storycontentdata.organization_name.value_counts()/sum(storycontentdata.organization_name.value_counts())*100)
fig = sns.barplot(temp.index,temp.organization_name)
fig.set_title('Organization Name(Percentage)')
fig.set_xlabel('Organization Name')
fig.set_ylabel('Percentage')

plt.xticks(rotation=90)
#Setting y limits
plt.ylim([0,100])
plt.show()
#Observation: There are nearly 11 publisher organizations. However, a large % of the stories 
#on the website are those from Pratham books.
#View how many reading levels are present in the data and distribution of stories amongst those.
plt.figure(figsize = (5,7))

temp = pd.DataFrame(storycontentdata.reading_level_cat.value_counts()/sum(storycontentdata.reading_level_cat.value_counts())*100)
fig = sns.barplot(temp.index,temp.reading_level_cat)
fig.set_title('Reading Level (Percentage)')
fig.set_xlabel('Reading Level')
fig.set_ylabel('Percentage')

#Setting y limits
plt.ylim([0,100])
plt.show()
#Observation: Close to 50% of the stories are of reading level 1. 
#View the number of reads for the stories to get an idea of readership of the books. 
#Besides being best books, perhaps these highly read books may be well tagged and hence
#easier to find than others 
plt.figure(figsize = (10,7))
sns.boxplot(storycontentdata.reads)
#Observation: Several books have not even been read once. Several have been read about 2-3K times. 
#Some books enjoy very high readership.<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09712982322752328, 'fraction_numerical': 0.014255844896407526, 'mean_word_length': 6.228021978021978, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '131469', 'license': 'MIT', 'n_tokens_mistral': 1692, 'n_tokens_neox': 1587, 'n_words': 529, 'path': 'python/notebooks/Story Data Exploration.ipynb', 'repo_name': 'yesemsanthoshkumar/PrathamBooks-Sprint-2018', 'size': '5261'}