<jupyter_start><jupyter_text>Exploratory Analysis<jupyter_code>from pandas import concat, DataFrame
from sklearn.cross_validation import train_test_split
from sklearn import ensemble

from context import *
from util.dfmgmt import initSet, wrangle
from util.polarize import sentiment
from util.plots import boxplotify, pairplotify, meanLine<jupyter_output><empty_output><jupyter_text>Plot characteristics and global variables<jupyter_code>%matplotlib inline<jupyter_output><empty_output><jupyter_text>Import dataset<jupyter_code>df = initSet()
df = df[df['decade'] != 2010]
# Check head
df.head(10)<jupyter_output><empty_output><jupyter_text>Basic Exploratory Analysis<jupyter_code># Basic descriptions of the distribution of the data
df.describe()
# # Get pair plot for major features of the dataset
pairplotify(
    df[
        ['year',
         'density',
         'unique_words',
         'total_curses', 
         'sentiment']
    ]
)
# Drop both year and decade
dropList = ['most_used_term']
removeList = ['decade', 'year', 'charted']

df, features = wrangle(df, dropList, removeList, True)

df['sentiment_polarity'] = df.apply(sentiment, axis=1)  # classify sentiment score
df.head(10)
# Plot features against time
features = [
    'density', 'unique_words', 'most_used_freq', 'verbs', 'nouns', 'adjectives', 'sentiment'
]
boxplotify(
    df, feature='density', path='densityXtime', title="Density of lyrics over decades"
)
boxplotify(
    df, feature='unique_words', path='unique_wordsXtime', 
    title="Total number of unique words in lyrics over decades"
)
boxplotify(
    df, feature='verbs', path='verbsXtime', 
    title="Total number of verbs in lyrics over decades"
)
boxplotify(
    df, feature='nouns', path='nounsXtime', 
    title="Total number of nouns in lyrics over decades"
)
boxplotify(
    df, feature='adjectives', path='adjectivesXtime', 
    title="Total number of adjectives in lyrics over decades")
boxplotify(
    df, feature='sentiment', path='sentimentXtime', 
    title="Sentiment score in lyrics over decades"
)
boxplotify(
    df, feature='most_used_freq', path='most_used_freqXtime', 
    title="Most used word frequency in lyrics over decades"
)
meanLine(
    df, feature='total_curses', path='profanityXtime', 
    title="Profanity in lyrics over decades"
)<jupyter_output><empty_output><jupyter_text>Use model to view the best feature over decades<jupyter_code>df = initSet()
def bestFeat(decade, df):
    
    # Keep both year and decade
    df_byDecade = df[df['decade'] == decade]
    dropList = ['most_used_term', 'reading_score']
    removeList = ['charted', 'year', 'decade']

    df_byDecade, features = wrangle(df_byDecade, dropList, removeList, True)

    # set X and y and build model
    X = df_byDecade[features]
    y = df_byDecade['charted']

    # Create separate training and test sets with 60/40 train/test split
    X_train, X_test, y_train, y_test = train_test_split(
            X, y, test_size=0.4, random_state=1000
    )

    # train model with classifier and parameter determined best
    clf = ensemble.RandomForestClassifier(
        n_estimators=256, min_samples_leaf=2, max_depth=8
    )
    clf.fit(X_train, y_train)
    
    # Plot importances for all features
    features = X.columns
    feature_importances = clf.feature_importances_

    features_df = DataFrame(
        {
            'Year': decade,
            'Features': features,
            'Importance Score': feature_importances
        }
    )

    features_df.sort_values(
        'Importance Score', inplace=True, ascending=False)

    return features_df.head(1)

newFrame = []

for decade in xrange(1960, 2010, 10):
    newFrame.append(bestFeat(decade, df))

newDf = concat(newFrame)
newDf = newDf[['Year', 'Features', 'Importance Score']]
newDf.reset_index(drop=True)
import matplotlib.pyplot as plt
from pandas import Series

freq_series = Series.from_array(newDf['Importance Score'])
plt.figure(figsize=(12, 8))

ax = freq_series.plot(kind='bar', color='rrgbb')
ax.set_title("Best Features Over Decades")
ax.set_xlabel("Decade")
ax.set_ylabel("Importance Score")

ax.set_xticklabels(newDf['Year'])

rects = ax.patches

# Now make some labels
labels = newDf['Features']
for rect, label in zip(rects, labels):
    ax.text(
        rect.get_x() + rect.get_width()/2, 
        rect.get_height(), label, 
        ha='center', va='bottom'
    )<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11390968913092807, 'fraction_numerical': 0.008849557522123894, 'mean_word_length': 4.415233415233415, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '608516', 'license': 'MIT', 'n_tokens_mistral': 1496, 'n_tokens_neox': 1438, 'n_words': 364, 'path': 'src/assets/models/notebooks/exploratory-analysis.ipynb', 'repo_name': 'kug3lblitz/Heat-Replay', 'size': '4407'}