<jupyter_start><jupyter_text>Clustering Crypto<jupyter_code># Author: <NAME>
# Homework: 13-AWS Lex
# Date: 26 November 2019
# Columbia Engineering Financial Engineering
import requests
import pandas as pd
import matplotlib.pyplot as plt
import hvplot.pandas
import plotly.express as px
from sklearn.preprocessing import StandardScaler, MinMaxScaler
from path import Path
from sklearn.decomposition import PCA
from sklearn.cluster import KMeans<jupyter_output><empty_output><jupyter_text>Fetching Cryptocurrency Data<jupyter_code>url = "https://min-api.cryptocompare.com/data/all/coinlist"
# Creating a DataFrame
file_path = Path("crypto_data.csv")

df_crypto = pd.read_csv(file_path, index_col='Unnamed: 0')

df_crypto.head()<jupyter_output><empty_output><jupyter_text>Data Preprocessing<jupyter_code>df_crypto.isnull().sum()
df_crypto.dropna(inplace=True)
df_crypto.isnull().sum()
df_crypto.head()
# Keep only cryptocurrencies that are on trading
df_crypto.drop(df_crypto[df_crypto['IsTrading'] == False].index, inplace=True)
# Keep only cryptocurrencies with a working algorithm
df_crypto.drop(df_crypto[df_crypto['TotalCoinsMined'] == 0].index, inplace=True)
# Remove the "IsTrading" column
df_crypto.drop("IsTrading", axis=1, inplace=True)
# Removing rows with at least 1 null value
df_crypto.dropna(inplace=True)
# Removing rows with cryptocurrencies without coins mined
name = df_crypto.copy()
# Dropping rows where there are 'N/A' text values
df_crypto = df_crypto[df_crypto!='N/A']
# Fetching the cryptocurrencies names prior to drop them from df_crypto
name = df_crypto.copy()
# Removig the cryptocurrency name since it's not going to be used on the clustering algorithm
df_crypto.drop('CoinName', axis=1, inplace=True)
# Creating dummies variables for text features
crypto_binary_encoded= pd.get_dummies(df_crypto, columns=['Algorithm', 'ProofType'])
# Standardizing data
crypto_scaled = StandardScaler().fit_transform(crypto_binary_encoded)

crypto_scaled<jupyter_output><empty_output><jupyter_text>Reducing Dimensions Using PCA<jupyter_code># Using PCA to reduce dimension to 3 principal components
pca = PCA(n_components=3)
coin_pca = pca.fit_transform(crypto_scaled)
# Creating a DataFrame with the principal components data
df_crypto_pca= pd.DataFrame(data=crypto_pca, columns=["PC1", "PC2", "PC3"], index= df_crypto.index)

df_crypto_pca.head()<jupyter_output><empty_output><jupyter_text>Clustering Crytocurrencies Using K-Means Finding the Best Value for `k` Using the Elbow Curve<jupyter_code>inertia = []
k = list(range(1, 11))

# Calculate the inertia for the range ok k values
for i in k:
    km = KMeans(n_clusters=i, random_state=0)
    km.fit(df_crypto_pca)
    inertia.append(km.inertia_)

# Create the Elbow Curve using hvPlot
elbow_curve = {"k": k, "inertia": inertia}
df_curve = pd.DataFrame(elbow_curve)
df_curve.hvplot.line(x="k", y="inertia", xticks=k, title="Elbow Curve")<jupyter_output><empty_output><jupyter_text>Running K-Means with `k=`<jupyter_code># Initialize the K-Means model
model = KMeans(n_clusters=4, random_state=0)
# Fit the model
model.fit(df_crypto_pca)
# Predict clusters
predictions = model.predict(df_crypto_pca)
# Creating a new DataFrame including predicted clusters and cryptocurrencies features
df_crypto_pca["class"] = model.labels_
df_crypto_pca.head()<jupyter_output><empty_output><jupyter_text>Visualizing Results 3D-Scatter with Clusters<jupyter_code># Creating a 3D-Scatter with the PCA data and the clusters
fig_scatter = px.scatter_3d(df_crypto_pca, x="PC1", y="PC2", z="PC3", color="class", symbol="class", width=650)
fig_scatter.show()<jupyter_output><empty_output><jupyter_text>Table of Tradable Cryptocurrencies<jupyter_code># Table with tradable cryptos
clustered = pd.merge(name, df_crypto_pca, left_index=True, right_index=True)
clustered.head()<jupyter_output><empty_output><jupyter_text>Scatter Plot with Tradable Cryptocurrencies<jupyter_code># Scaling data to create the scatter plot
features = clustered.copy()

#selecct columns to scale
columns_names = ['TotalCoinsMined', 'TotalCoinSupply']
features_1 = features[columns_names]

#scale with MinMaxScaler scaler
scaler = MinMaxScaler().fit(features_1.values)
features_1 = scaler.transform(features_1.values)

#amend columns with scaled values
features[columns_names] = features_1
features

# Plot the scatter with x="TotalCoinsMined" and y="TotalCoinSupply"
features.hvplot.scatter(x="TotalCoinsMined", y="TotalCoinSupply")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10307898259705489, 'fraction_numerical': 0.008478357875948238, 'mean_word_length': 7.94810379241517, 'pattern_counts': {'":': 2, '<': 38, '<?xml version=': 0, '>': 38, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '80429', 'license': 'Unlicense', 'n_tokens_mistral': 1601, 'n_tokens_neox': 1527, 'n_words': 363, 'path': 'crypto_clustering.ipynb', 'repo_name': 'jrmatos92/AWS_Lex_HW', 'size': '4490'}