<jupyter_start><jupyter_text>Check Class Imbalance<jupyter_code>fig, ax = plt.subplots(figsize=(8, 4))
df['iris_type'].value_counts(normalize=True).plot.barh(ax=ax)

ax.set_title("There is no class imbalance\n")
ax.set_xlabel("\n% of cases")
ax.set_ylabel("Class\n")

plt.savefig("figures/01-class-distribution.png", bbox_inches='tight')
;<jupyter_output><empty_output><jupyter_text>Variation across groups<jupyter_code>fig, ax = plt.subplots(figsize=(10, 6))
df.groupby('iris_type').mean().T.plot.barh(ax=ax)

ax.set_title("""Very litte variation across groups in Sepal Width, \nbut Petal Length might be an important predictor \n""")
ax.set_xlabel("\n Mean Values")
ax.set_ylabel("Features \n")

plt.savefig("figures/02-across-groups-variation.png", bbox_inches='tight');<jupyter_output><empty_output><jupyter_text>ANOVA<jupyter_code>from sklearn.feature_selection import SelectKBest, f_classif
skb = SelectKBest(k='all', score_func=f_classif)

X = df.copy().drop(['iris_type'], axis=1).values
y = df.copy().loc[:, 'iris_type'].replace({'setosa': 0, 'versicolor': 1, 'virginica': 2}).values

skb.fit(X, y)

anova_results = \
pd.DataFrame({
    'F-score': skb.scores_,
    'P-value': skb.pvalues_}, index=df.columns[:4])
fig, ax = plt.subplots(figsize=(8, 4))
anova_results.plot.barh(ax=ax)

ax.set_title("ANOVA Results: All predictors are significant \n Petal Length and width are most important \n")
ax.set_xlabel("\n Score")
ax.set_ylabel("Predictor \n")

plt.savefig("figures/03-ANOVA-results.png", bbox_inches='tight');<jupyter_output><empty_output><jupyter_text>Principal Components Analysis<jupyter_code>from sklearn.decomposition import PCA
pca = PCA(n_components=2)
pca.fit(X, y)
(pd.Series(pca.explained_variance_ratio_, 
           index=['Component 1', 'Component 2'])
 .plot.barh(title="2-component solution explains >95% of the variance \n", figsize=(8, 4))
)

plt.savefig("figures/04-PCA-results.png", bbox_inches='tight');
X_2dimensions = pd.DataFrame(pca.transform(X), columns=['C1', 'C2'])
fig, ax = plt.subplots(figsize=(7, 4))
X_2dimensions.plot.scatter(x='C1', y='C2', c=y,  cmap='winter', ax=ax)

ax.set_title("Visualize Classification Boundary in 2-dimensions \n")
ax.set_xlabel("\n Component 1")
ax.set_ylabel("Component 2 \n")
ax.text(7, 0, """
Setosa will be easily classifiable, 
but we will find it hard to distinguish 
between some virginica and versicolor""", fontsize=16)

plt.savefig("figures/05-classification-boundary.png", bbox_inches='tight');<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16892430278884463, 'fraction_numerical': 0.017928286852589643, 'mean_word_length': 6.822429906542056, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '357363', 'license': 'MIT', 'n_tokens_mistral': 1012, 'n_tokens_neox': 948, 'n_words': 176, 'path': 'code/dk-iris-pipeline/notebooks/03-Explore.ipynb', 'repo_name': 'dushyantkhosla/airflow4ds', 'size': '2510'}