<jupyter_start><jupyter_text>Pokemon Statistical AnalysisThe purpose of this project is to experiment with some Exploratory data analysis with some Pokèmon statistics.In this notebook I will frequently use the abbreviation "gen" when referring to "generation".  Imports<jupyter_code>import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline<jupyter_output><empty_output><jupyter_text>Read In Data<jupyter_code>poke_df = pd.read_csv('./data/pokemon.csv')
poke_df.head()
poke_df.describe()
poke_df.columns<jupyter_output><empty_output><jupyter_text>Exploratory Data Analysis (EDA) For Each Generation  Gen 1 Charts<jupyter_code>gen_1 = poke_df.query('Generation==1')
gen_1.head()
gen_1_means = gen_1.groupby('Type 1').mean(numeric_only=True)
gen_1_means.drop(columns=['#', 'Generation', 'Legendary'], inplace=True)
gen_1_means.head()
gen_1_counts = gen_1.groupby('Type 1').count().filter('#')
fig = gen_1_counts.plot.bar(legend=False, title='Pokèmon with Type 1').get_figure()
fig.savefig('images/gen_1/gen_1_count.jpg')
fig, axs = plt.subplots(2, 2, sharey=True, tight_layout=True)
axs[0][0].hist(gen_1['HP'], label='HP')
axs[0][1].hist(gen_1['Attack'], label='Attack', color='red')
axs[1][0].hist(gen_1['Speed'], label='Speed', color='green')
axs[1][1].hist(gen_1['Defense'], label='Defense', color='purple')
fig.legend()
fig.savefig('images/gen_1/gen_1_charts.jpg')<jupyter_output><empty_output><jupyter_text>Gen 2 Charts<jupyter_code>gen_2 = poke_df.query('Generation==2')
gen_2.head()
gen_2_means = gen_2.groupby('Type 1').mean(numeric_only=True)
gen_2_means.drop(columns=['#', 'Generation', 'Legendary'], inplace=True)
gen_2_means.head()
gen_2_counts = gen_2.groupby('Type 1').count().filter('#')
fig = gen_2_counts.plot.bar(legend=False, title='Pokèmon with Type 1').get_figure()
fig.savefig('images/gen_2/gen_2_counts.jpg')
fig, axs = plt.subplots(2, 2, sharey=True, tight_layout=True)
axs[0][0].hist(gen_2['HP'], label='HP')
axs[0][1].hist(gen_2['Attack'], label='Attack', color='red')
axs[1][0].hist(gen_2['Speed'], label='Speed', color='green')
axs[1][1].hist(gen_2['Defense'], label='Defense', color='purple')
fig.legend()
fig.savefig('images/gen_2/gen_2_charts.jpg')<jupyter_output><empty_output><jupyter_text>Gen 3 Charts<jupyter_code>gen_3 = poke_df.query('Generation==3')
gen_3.head()
gen_3_means = gen_3.groupby('Type 1').mean(numeric_only=True)
gen_3_means.drop(columns=['#', 'Generation', 'Legendary'], inplace=True)
gen_3_means.head()
gen_3_counts = gen_3.groupby('Type 1').count().filter('#')
fig = gen_3_counts.plot.bar(legend=False, title='Pokèmon with Type 1').get_figure()
fig.savefig('images/gen_3/gen_3_counts.jpg')
fig, axs = plt.subplots(2, 2, sharey=True, tight_layout=True)
axs[0][0].hist(gen_3['HP'], label='HP')
axs[0][1].hist(gen_3['Attack'], label='Attack', color='red')
axs[1][0].hist(gen_3['Speed'], label='Speed', color='green')
axs[1][1].hist(gen_3['Defense'], label='Defense', color='purple')
fig.legend()
fig.savefig('images/gen_3/gen_3_charts.jpg')<jupyter_output><empty_output><jupyter_text>Gen 4 Charts<jupyter_code>gen_4 = poke_df.query('Generation==4')
gen_4.head()
gen_4_means = gen_4.groupby('Type 1').mean(numeric_only=True)
gen_4_means.drop(columns=['#', 'Generation', 'Legendary'], inplace=True)
gen_4_means.head()
gen_4_counts = gen_4.groupby('Type 1').count().filter('#')
fig = gen_4_counts.plot.bar(legend=False, title='Pokèmon with Type 1').get_figure()
fig.savefig('images/gen_4/gen_4_counts.jpg')
fig, axs = plt.subplots(2, 2, sharey=True, tight_layout=True)
axs[0][0].hist(gen_4['HP'], label='HP')
axs[0][1].hist(gen_4['Attack'], label='Attack', color='red')
axs[1][0].hist(gen_4['Speed'], label='Speed', color='green')
axs[1][1].hist(gen_4['Defense'], label='Defense', color='purple')
fig.legend()
fig.savefig('images/gen_4/gen_4_charts.jpg')<jupyter_output><empty_output><jupyter_text>Gen 5 Charts<jupyter_code>gen_5 = poke_df.query('Generation==5')
gen_5.head()
gen_5_means = gen_5.groupby('Type 1').mean(numeric_only=True)
gen_5_means.drop(columns=['#', 'Generation', 'Legendary'], inplace=True)
gen_5_means.head()
gen_5_counts = gen_5.groupby('Type 1').count().filter('#')
fig = gen_5_counts.plot.bar(legend=False, title='Pokèmon with Type 1').get_figure()
fig.savefig('images/gen_5/gen_5_counts.jpg')
fig, axs = plt.subplots(2, 2, sharey=True, tight_layout=True)
axs[0][0].hist(gen_5['HP'], label='HP')
axs[0][1].hist(gen_5['Attack'], label='Attack', color='red')
axs[1][0].hist(gen_5['Speed'], label='Speed', color='green')
axs[1][1].hist(gen_5['Defense'], label='Defense', color='purple')
fig.legend()
fig.savefig('images/gen_5/gen_5_charts.jpg')<jupyter_output><empty_output><jupyter_text>Gen 6 Charts<jupyter_code>gen_6 = poke_df.query('Generation==6')
gen_6.head()
gen_6_means = gen_6.groupby('Type 1').mean(numeric_only=True)
gen_6_means.drop(columns=['#', 'Generation', 'Legendary'], inplace=True)
gen_6_means.head()
gen_6_counts = gen_6.groupby('Type 1').count().filter('#')
fig = gen_6_counts.plot.bar(legend=False, title='Pokèmon with Type 1').get_figure()
fig.savefig('images/gen_6/gen_6_counts.jpg')
fig, axs = plt.subplots(2, 2, sharey=True, tight_layout=True)
axs[0][0].hist(gen_6['HP'], label='HP')
axs[0][1].hist(gen_6['Attack'], label='Attack', color='red')
axs[1][0].hist(gen_6['Speed'], label='Speed', color='green')
axs[1][1].hist(gen_6['Defense'], label='Defense', color='purple')
fig.legend()
fig.savefig('images/gen_6/gen_6_charts.jpg')<jupyter_output><empty_output><jupyter_text>EDA Cross GenerationFor this section each generation has its own color.| Color | Generation |---------- | --------Blue | Gen 1 |Gold | Gen 2 |Orange | Gen 3 |Purple | Gen 4 |Grey | Gen 5 |Cyan | Gen 6 |<jupyter_code>fig, axs = plt.subplots(2, 3, sharey=True, tight_layout=True)
fig.suptitle('HP')
axs[0][0].hist(gen_1['HP'], label='Gen 1', color='blue')
axs[0][1].hist(gen_2['HP'], label='Gen 2', color='gold')
axs[0][2].hist(gen_3['HP'], label='Gen 3', color='orange')
axs[1][0].hist(gen_4['HP'], label='Gen 4', color='purple')
axs[1][1].hist(gen_5['HP'], label='Gen 5', color='grey')
axs[1][2].hist(gen_6['HP'], label='Gen 6', color='cyan')
fig.legend()
fig.savefig('images/hp_compare.jpg')
fig, axs = plt.subplots(2, 3, sharey=True, tight_layout=True)
fig.suptitle('Attack')
axs[0][0].hist(gen_1['Attack'], label='Gen 1', color='blue')
axs[0][1].hist(gen_2['Attack'], label='Gen 2', color='gold')
axs[0][2].hist(gen_3['Attack'], label='Gen 3', color='orange')
axs[1][0].hist(gen_4['Attack'], label='Gen 4', color='purple')
axs[1][1].hist(gen_5['Attack'], label='Gen 5', color='grey')
axs[1][2].hist(gen_6['Attack'], label='Gen 6', color='cyan')
fig.legend()
fig.savefig('images/attack_compare.jpg')
fig, axs = plt.subplots(2, 3, sharey=True, tight_layout=True)
fig.suptitle('Defense')
axs[0][0].hist(gen_1['Defense'], label='Gen 1', color='blue')
axs[0][1].hist(gen_2['Defense'], label='Gen 2', color='gold')
axs[0][2].hist(gen_3['Defense'], label='Gen 3', color='orange')
axs[1][0].hist(gen_4['Defense'], label='Gen 4', color='purple')
axs[1][1].hist(gen_5['Defense'], label='Gen 5', color='grey')
axs[1][2].hist(gen_6['Defense'], label='Gen 6', color='cyan')
fig.legend()
fig.savefig('images/defense_compare.jpg')
fig, axs = plt.subplots(2, 3, sharey=True, tight_layout=True)
fig.suptitle('Speed')
axs[0][0].hist(gen_1['Speed'], label='Gen 1', color='blue')
axs[0][1].hist(gen_2['Speed'], label='Gen 2', color='gold')
axs[0][2].hist(gen_3['Speed'], label='Gen 3', color='orange')
axs[1][0].hist(gen_4['Speed'], label='Gen 4', color='purple')
axs[1][1].hist(gen_5['Speed'], label='Gen 5', color='grey')
axs[1][2].hist(gen_6['Speed'], label='Gen 6', color='cyan')
fig.legend()
fig.savefig('images/speed_compare.jpg')<jupyter_output><empty_output><jupyter_text>When comparing the histogram charts of different Pokèmon stats you can notice a strange behavior that Gens 1, 3, and 5 havesimilar distributions and 2, 4, and 6 have similar distributions. This is interesting because it also looks like Gen 1, 3, and5 have a higher top bin than on most stats. This would mean that more of the Pokèmon have lower stats, but the maxstats for the generation are higher.For Gens 2, 4, and 6 we can notice that the distribution is more even throughout the stats but the max stats for these gensare lower than the other generations.Now I want to compare the mean stats for each generation.<jupyter_code>poke_means = poke_df.groupby('Generation').mean(numeric_only=True)
poke_means.drop(columns=['#', 'Legendary'], inplace=True)
poke_means.head(6)
fig = poke_means.plot.bar(y='HP', color=['blue', 'gold', 'orange', 'purple', 'grey', 'cyan'], legend=False, title='HP').get_figure()
fig.savefig('images/hp_means.jpg')
fig = poke_means.plot.bar(y='Attack', color=['blue', 'gold', 'orange', 'purple', 'grey', 'cyan'], legend=False, title='Attack').get_figure()
fig.savefig('images/attack_means.jpg')
fig = poke_means.plot.bar(y='Defense', color=['blue', 'gold', 'orange', 'purple', 'grey', 'cyan'], legend=False, title='Defense').get_figure()
fig.savefig('images/defense_means.jpg')
fig = poke_means.plot.bar(y='Speed', color=['blue', 'gold', 'orange', 'purple', 'grey', 'cyan'], legend=False, title='Speed').get_figure()
fig.savefig('images/speed_means.jpg')<jupyter_output><empty_output><jupyter_text>From the above charts were I compared the mean of each stat per generation we can see that there isn't as muchof a pattern as we saw with the histograms. This shows that even though the histograms are off that there arejust a few outliers in each generation but the majority of Pokèmon have similar stats across generations.This next chart is going to look a little crazy but stick with me.<jupyter_code>gen_type = poke_df.groupby(['Generation', 'Type 1']).count().filter('#').unstack('Type 1')
fig = gen_type.plot(kind='bar', stacked=True, legend=False, rot=0, title='Type per Generation').get_figure()
fig.legend(loc='center left', bbox_to_anchor=(1.0, 0.5))
fig.savefig('images/type_per_gen.jpg')<jupyter_output><empty_output><jupyter_text>I warned you this chart was going to look a little crazy, but we can notice some interesting things from this chart.We notice the same correlation pattern between Gens 1, 3, and 5 having, in some cases significantly, more Pokèmonthan the other generations. Knowing this, we can revisit some of the earlier charts we generated with a little morecontext.  SKLEARN time!! 🎉In this section I'm wanting to try to use sklearn to try to cluster and/or classify the data.  Classification<jupyter_code>sk_df = poke_df.drop(columns=['#', 'Type 2', 'Legendary', 'Name'], axis=1)
sk_df.head()
types = list(sk_df['Type 1'].unique())
type_mapping = {}
for x in range(len(types)):
    type_mapping[types[x]] = x
sk_df['Type 1'] = sk_df['Type 1'].map(type_mapping)
sk_df.head()
from sklearn.model_selection import train_test_split
x_train, x_test= train_test_split(sk_df)

train_types = x_train['Type 1']
del x_train['Type 1']

test_types = x_test['Type 1']
del x_test['Type 1']<jupyter_output><empty_output><jupyter_text>Random Forest Classifier<jupyter_code>from sklearn.ensemble import RandomForestClassifier

clf = RandomForestClassifier(max_depth=7, random_state=0)
clf.fit(x_train, train_types)
rf_score = clf.score(x_test, test_types)
print(f'Score: {rf_score}')<jupyter_output>Score: 0.295<jupyter_text>Logistic Regression<jupyter_code>from sklearn.linear_model import LogisticRegression

clf = LogisticRegression(random_state=0, max_iter=10000).fit(x_train, train_types)
lr_score = clf.score(x_test, test_types)
print(f'Score: {lr_score}')<jupyter_output>Score: 0.22<jupyter_text>Linear Support Vector Classification<jupyter_code>from sklearn.svm import LinearSVC
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler

clf = make_pipeline(StandardScaler(), LinearSVC(random_state=0))
clf.fit(x_train, train_types)
svc_score = clf.score(x_test, test_types)
print(f'Score: {svc_score}')<jupyter_output>Score: 0.21<jupyter_text>Clustering  K Means<jupyter_code>from sklearn.cluster import KMeans
df_kmeans = sk_df.copy()
kmeans = KMeans(n_clusters=18, random_state=0).fit(df_kmeans[['Total', 'Generation']])
print(kmeans.cluster_centers_)
df_kmeans['pred'] = kmeans.labels_
df_kmeans.head(10)
df_kmeans.plot.scatter(x='Generation', y='Total', colormap='viridis', c='pred')<jupyter_output>/Users/thomas/.virtualenvs/notebook/lib/python3.9/site-packages/pandas/plotting/_matplotlib/tools.py:400: MatplotlibDeprecationWarning: 
The is_first_col function was deprecated in Matplotlib 3.4 and will be removed two minor releases later. Use ax.get_subplotspec().is_first_col() instead.
  if ax.is_first_col():
{'fraction_non_alphanumeric': 0.18316322493868187, 'fraction_numerical': 0.028878866998971436, 'mean_word_length': 8.397769516728625, 'pattern_counts': {'":': 0, '<': 61, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '249015', 'license': 'MIT', 'n_tokens_mistral': 4987, 'n_tokens_neox': 4773, 'n_words': 924, 'path': 'PokemonStatistics/notebook.ipynb', 'repo_name': 'lazarust/JupyterNotebooks', 'size': '12639'}