<jupyter_start><jupyter_text>Data Setup  Packages<jupyter_code># notebook install
# ! pip install umap-learn

# local
pip install umap-learn

# imports
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt

# scikit
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler
from sklearn.manifold import TSNE
from sklearn.cluster import KMeans, DBSCAN

# umap
from umap import UMAP<jupyter_output><empty_output><jupyter_text>Tools  Fill drop function<jupyter_code>def fillndrop(x,parameter,df,manual=None):
    """input is a list of column title, and if you want the mean, mode, or drop. 
    After replacing columns with nulls, function will drop remaining values
    """
    if parameter == 'mode':
        for i in x:
            df[i] = df[i].fillna(df[i].mode()[0])
    if parameter == 'mean':
        for i in x:
            df[i] = df[i].fillna(df[i].mean())
    if parameter == 'drop':
        for i in x:
            if i in df:
                df.drop(i,axis=1,inplace=True)
                
    if parameter == 'manual':
        for i in x:
            df[i] = df[i].fillna(manual)<jupyter_output><empty_output><jupyter_text>explained variance bar plot function<jupyter_code># quick function to construct the barplot easily
def ev_plot(ev):
  y = list(ev)
  x = list(range(1,len(ev)+1))
  return x, y<jupyter_output><empty_output><jupyter_text>Data Exploration  Description & Info<jupyter_code>csv = pd.read_csv("stock-fundamentals.csv")
stonks_all = pd.DataFrame(csv)
stonks = stonks_all.set_index("ticker")
stonks.info()
stonks.head().T
stonks.describe().T<jupyter_output><empty_output><jupyter_text>Null Values<jupyter_code>nulls = stonks.isnull().sum().sort_values(ascending=False).to_frame()
nulls = nulls[nulls[0] != 0]
nulls
plt.figure(figsize=(10, 8))
plt.title("Null values")
sns.heatmap(stonks.isnull(),yticklabels=False,cbar=False)
plt.show()<jupyter_output><empty_output><jupyter_text>Data Visualization<jupyter_code>stonks_corr = stonks.select_dtypes("number").corr()
plt.figure(figsize=(12, 10))
plt.title("Correlation Heatmap")
sns.heatmap(stonks_corr, cmap="inferno")
plt.show()
for column in stonks.drop(['Quarter end','Split factor'], axis=1):
    plt.hist(stonks[column])
    plt.title(column)
    plt.show()<jupyter_output><empty_output><jupyter_text>Data Preparation  Feature Engineering<jupyter_code>stonks['quarter_end_month'] = stonks['quarter_end'].apply(lambda x: x[5:7])
stonks['P/S ratio'] = stonks['Shares']*stonks['Price'] / stonks['Revenue']<jupyter_output><empty_output><jupyter_text>Data Cleaning<jupyter_code>nulls = stonks.isnull().sum().sort_values(ascending=False).to_frame()
nulls = nulls[nulls[0] != 0]
# nulls
# nulls.index.to_list()
# # quick data analyis functions
# stonks[""].unique()
# stonks[""].value_counts()
# mean fill
fillndrop(['ROA', # 1 Nulls
           'Free cash flow per share', # 2 Nulls
           'Cash from operating activities', # 2 Nulls
           'Cash from investing activities', # 2 Nulls
           'Cash from financing activities', # 2 Nulls
           'Revenue', # 3 Nulls
           'Asset turnover', # 7 Nulls
           'Price low', # 43 Nulls
           'Price high', # 43 Nulls
           'Price', # 43 Nulls
           'Equity to assets ratio', # 43 Nulls
           'P/S ratio', # 43 nulls
           'Long-term debt to equity ratio', # had 75 values equal to 0
           'P/B ratio', # 84 Nulls
           'ROE', # 63 Nulls
           'P/E ratio', # 139 Nulls
           'Current ratio', # 122 Nulls
           'Current Assets', # 122 Nulls
           'Current Liabilities', # 122 Nulls
           'Net margin', # 103 Nulls
           'Dividend payout ratio', # 102 Nulls
           ],'mean',stonks)
# drop variable
fillndrop(['Quarter end', # 755 Nulls
           'Split factor', # 755 value of 1
           'quarter_end' # Feature Engineering
           ],'drop',stonks)
# Dropping Berkshire Hathaway Inc. Class A stock because it is an extreme outlier
stonks.drop('BRK', axis = 0, inplace=True)<jupyter_output><empty_output><jupyter_text>Data Scaling<jupyter_code>stonks_num = stonks.select_dtypes("number")

scaled = StandardScaler()
stonks_scaled = scaled.fit_transform(stonks_num)<jupyter_output><empty_output><jupyter_text>Data Modeling  K Means<jupyter_code># top metrics for stocks based on investors https://www.fool.com/investing/2018/03/21/9-essential-metrics-all-smart-investors-should-kno.aspx
top_col = [
           'ROE',
           'P/E ratio',
           'P/B ratio',
           'P/S ratio',
           'Free cash flow per share',
           'Dividend payout ratio',
           'Long-term debt to equity ratio',
        ]

stonks_top = stonks_num.loc[:,top_col]
top_scaled = scaled.fit_transform(stonks_top)
ss= []
for k in range(2,30):
    km = KMeans(n_clusters=k)
    lab_sc = km.fit_predict(stonks_scaled)
    ss.append(km.inertia_)

plt.title("All Columns \nInertia v # of clusters")
sns.lineplot(x=range(2,30), y=ss)
plt.show()

# k =KMeans(7)
# k_pred = k.fit_predict(stonks_scaled)
# stonks["k"] =k_pred
# k_centers = k.cluster_centers_<jupyter_output><empty_output><jupyter_text>PCA<jupyter_code>pca = PCA()
pcs = pca.fit_transform(stonks_scaled)
# pcs = pca.fit_transform(top_scaled)
# explained variance

varexp = pca.explained_variance_ratio_
# sns.lineplot(range(1, len(varexp)+1), varexp)

# cumulative explained  variance

plt.title("Cumulative Explained Variance \n85% EV = 17 Components")
plt.plot(range(1, len(varexp)+1), np.cumsum(varexp))
plt.axhline(.85, color="red")
plt.show()


pca_85 = PCA(.85)
pcs_85 = pca_tsne.fit_transform(stonks_scaled)

# Number of (observation, components)
pcs_85.shape

x, y = ev_plot(pca.explained_variance_)
sns.barplot(x, y)
plt.show()
comps = pca.components_

cols = ["PC" + str(i) for i in range(1, len(comps)+1)]
loadings = pd.DataFrame(comps.T, columns=cols, index=stonks_num.columns)
# loadings

plt.figure(figsize=(10, 8))
plt.title("PCA component weight heatmap")
sns.heatmap(loadings, cmap="vlag")
plt.show()
pc2 = PCA(2, svd_solver = "full")
stonks_pc2 = pc2.fit_transform(stonks_scaled)

np.sum(pc2.explained_variance_ratio_)

stonks[['pc2_1','pc2_2']] = stonks_pc2
plt.title("""PCA_1 v PCA_2 \n 30.6% Explained Variance""")
plt.scatter (x=stonks.pc2_1, y=stonks.pc2_2)
plt.show()<jupyter_output><empty_output><jupyter_text>UMAP<jupyter_code>u = UMAP(
         n_neighbors= 5,
         min_dist=.3, random_state=820)
ue = u.fit_transform(stonks_top)

umap_df = pd.DataFrame(ue,columns=['x','y'])
ss= []
for k in range(2,30):
    km = KMeans(n_clusters=k)
    lab_sc = km.fit_predict(umap_df)
    ss.append(km.inertia_)

plt.title("Inertia score vs # of clusters for UMAP")
sns.lineplot(x=range(2,30), y=ss)
plt.show()
k =KMeans(5)
k_pred = k.fit_predict(umap_df)
stonks["k"] =k_pred
umap_df['k'] = stonks.reset_index()['k']

db = DBSCAN(eps=1.42, min_samples=10)
dbc = db.fit_predict(umap_df)
umap_df['dbc']= dbc
plt.figure(figsize=(9, 7))
plt.title("UMAP with KMeans clusters")
sns.scatterplot(x="x", y="y", hue= 'k', data= umap_df, palette="Accent")
plt.show()
plt.figure(figsize=(9, 7))
plt.title("UMAP with DBSCAN clusters")
sns.scatterplot(x="x", y="y", hue= 'dbc', data= umap_df, palette="Accent")
plt.show()<jupyter_output><empty_output><jupyter_text>TSNE<jupyter_code># two step process - use PCA to reduce, then tsne for embeddings (2d)
# to keep things simple, just random choice for 90%, this could be 80%, or 95%

pca_tsne = PCA(.85)
pcs_tsne = pca_tsne.fit_transform(stonks_scaled)

pcs_tsne.shape
tsne = TSNE(perplexity=50.0,n_iter=5000, init='pca', random_state= 820)
tsne.fit(pcs_tsne)
te = tsne.embedding_
tdata = pd.DataFrame(te, columns=["e1", "e2"])
ss= []
for k in range(2,30):
    km = KMeans(n_clusters=k)
    lab_sc = km.fit_predict(tdata)
    ss.append(km.inertia_)


plt.title("Inertia score vs # of clusters for TSNE")
sns.lineplot(x=range(2,30), y=ss)
plt.show()

db = DBSCAN(eps=2.45, min_samples=10)
dbc = db.fit_predict(tdata)
tdata['dbc']= dbc


k =KMeans(6)
k_pred = k.fit_predict(tdata)
stonks["k"] =k_pred
tdata['k'] = stonks.reset_index()['k']

plt.figure(figsize=(9, 7))
plt.title("TSNE with KMeans clusters")
sns.scatterplot(x="e1", y="e2", hue="k", data=tdata, palette= "Accent_r" )
plt.figure(figsize=(9, 7))
plt.title("TSNE with DBSCAN clusters")
sns.scatterplot(x="e1", y="e2", hue="dbc", data=tdata, palette= "Accent_r" )<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14670303605313093, 'fraction_numerical': 0.022296015180265655, 'mean_word_length': 4.347495244134432, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 53, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '638098', 'license': 'MIT', 'n_tokens_mistral': 3426, 'n_tokens_neox': 3231, 'n_words': 673, 'path': 'Clustering Stocks.ipynb', 'repo_name': 'conwuzurike/BA-820', 'size': '8432'}