<jupyter_start><jupyter_text>Clustering Crypto<jupyter_code># Initial imports
import requests
import pandas as pd
import matplotlib.pyplot as plt
# import hvplot.pandas
# import plotly.express as px
from sklearn.preprocessing import StandardScaler, MinMaxScaler
from sklearn.decomposition import PCA
from sklearn.cluster import KMeans

from urllib.request import Request, urlopen
import json

!pip install -U altair

# Amazon SageMaker and related imports
import sagemaker
import sagemaker.amazon.common as smac
from sagemaker.predictor import csv_serializer, json_deserializer
from sagemaker import get_execution_role
from sagemaker.amazon.amazon_estimator import get_image_uri
import boto3  # AWS Python sdk

%matplotlib inline<jupyter_output>Requirement already up-to-date: altair in c:\users\nites\anaconda3\envs\pyvizenv\lib\site-packages (4.1.0)
Requirement already satisfied, skipping upgrade: toolz in c:\users\nites\anaconda3\envs\pyvizenv\lib\site-packages (from altair) (0.10.0)
Requirement already satisfied, skipping upgrade: entrypoints in c:\users\nites\anaconda3\envs\pyvizenv\lib\site-packages (from altair) (0.3)
Requirement already satisfied, skipping upgrade: numpy in c:\users\nites\anaconda3\envs\pyvizenv\lib\site-packages (from altair) (1.18.5)
Requirement already satisfied, skipping upgrade: jsonschema in c:\users\nites\anaconda3\envs\pyvizenv\lib\site-packages (from altair) (3.2.0)
Requirement already satisfied, skipping upgrade: jinja2 in c:\users\nites\anaconda3\envs\pyvizenv\lib\site-packages (from altair) (2.11.2)
Requirement already satisfied, skipping upgrade: pandas>=0.18 in c:\users\nites\anaconda3\envs\pyvizenv\lib\site-packages (from altair) (0.24.2)
Requirement already satisfied, skipping upgrade: a[...]<jupyter_text>Fetching Cryptocurrency Data<jupyter_code># Use the following endpoint to fetch json data
url = "https://min-api.cryptocompare.com/data/all/coinlist"

# Create a DataFrame 
# HINT: You will need to use the 'Data' key from the json response, then transpose the DataFrame.

data = urlopen(Request(url)).read()
json_data = json.loads(data)
df_full = pd.DataFrame(json_data["Data"])
df_full.head()
# Alternatively, use the provided csv file:
# file_path = Path("Resources/crypto_data.csv")

# Create a DataFrame
df_transpose = df_full.T
df_transpose.head()<jupyter_output><empty_output><jupyter_text>Data Preprocessing<jupyter_code># Keep only necessary columns:
# 'CoinName','Algorithm','IsTrading','ProofType','TotalCoinsMined','TotalCoinSupply'

df = df_transpose[["CoinName", "Algorithm", "IsTrading", "ProofType", "TotalCoinsMined", "TotalCoinSupply"]]
df.head()
# Keep only cryptocurrencies that are trading

df = df.loc[df.IsTrading != False]
df.head()

# Keep only cryptocurrencies with a working algorithm
df = df.loc[df.Algorithm != "N/A"]
df.head()

# Remove the 'IsTrading' column
df = df.drop(columns = "IsTrading")
df.head()

# Remove rows with at least 1 null value
# to check the null values -->  df.isnull().sum()
df = df.dropna()
df.isnull().sum()
# Remove rows with cryptocurrencies having no coins mined
df = df.loc[df.TotalCoinsMined != 0]
df.head()

# Check Columns where there are 'N/A' text values
for i in df.columns:
    print (df[df[i] == "N/A"])


    

# Dropping rows where there are "N/A" text values in ProofType
df = df.loc[df["ProofType"].str.contains("N/A") == False]

# Dropping rows where there are "N/A" text values in TotalCoinsSupply
df = df.loc[df["TotalCoinSupply"].str.contains("N/A") == False]

# Store the 'CoinName'column in its own DataFrame prior to dropping it from crypto_df 

crypto_name = df.CoinName
crypto_name.head()

# Drop the 'CoinName' column since it's not going to be used on the clustering algorithm
cluster_df = df.drop(columns = "CoinName")
print(cluster_df.shape)
cluster_df.head()

# Create dummy variables for text features
cluster_df_dummy = pd.get_dummies(cluster_df[["Algorithm", "ProofType"]])
# cluster_df_dummy = pd.get_dummies(cluster_df["ProofType"])
cluster_df_dummy.head()

# Standardize data

cluster_df_scaled = StandardScaler().fit_transform(cluster_df_dummy)
cluster_df_scaled[0:5]<jupyter_output><empty_output><jupyter_text>Reducing Dimensions Using PCA<jupyter_code># Use PCA to reduce dimensions to 3 principal components

# Initialize PCA model
pca = PCA(n_components=2)

cluster_pca = pca.fit_transform(cluster_df_scaled)
# Create a DataFrame with the principal components data
df_cluster_pca = pd.DataFrame(data = cluster_pca, 
                              columns=["PC 1", "PC 2"],                              index = cluster_df.index
                              )
                    
print(cluster_pca.shape)
df_cluster_pca.head()
# Fetch the explained variance
pca.explained_variance_ratio_.sum()<jupyter_output><empty_output><jupyter_text>Clustering Crytocurrencies Using K-Means Finding the Best Value for `k` Using the Elbow Curve<jupyter_code>import altair as alt
inertia = []
k = list(range(1, 11))

# Calculate the inertia for the range of k values
for i in k:
    km = KMeans(n_clusters=i, random_state=0)
    km.fit(df_cluster_pca)
    inertia.append(km.inertia_)

# Create the Elbow Curve using hvPlot
elbow_data = {"k": k, "inertia": inertia}
df_elbow = pd.DataFrame(elbow_data)

alt.Chart(df_elbow, title = "Elbow Curve").mark_line().encode(
    x='k',
    y='inertia'
)<jupyter_output><empty_output><jupyter_text>Running K-Means with `k= 3`<jupyter_code># Initialize the K-Means model
model = KMeans(n_clusters=3, random_state=0)

# Fit the model
model.fit(df_cluster_pca)

# Predict clusters
predictions = model.predict(df_cluster_pca)

# Add the predicted class columns
df_cluster_pca["class"] = model.labels_
df_cluster_pca.head()
# df_cluster_pca.shape
# Create a new DataFrame including predicted clusters and cryptocurrencies features
df_clustered = pd.DataFrame({
    "Algorithm" : cluster_df.Algorithm,
    "ProofType": cluster_df.ProofType,
    "TotalCoinsMined": cluster_df.TotalCoinsMined,
    "TotalCoinSupply": cluster_df.TotalCoinSupply,
    "principal component 1": df_cluster_pca["PC 1"],
    "principal component 2": df_cluster_pca["PC 2"], 
    "CoinName": crypto_name,
    "Class": df_cluster_pca["class"]
    }, index = cluster_df.index
    )

df_clustered.head(20)<jupyter_output><empty_output><jupyter_text>Visualizing Results 3D-Scatter with Clusters<jupyter_code># Create a 3D-Scatter with the PCA data and the clusters
# fig = px.scatter_3d(
#     df_cluster_pca,
#     x="principal component 1",
#     y="principal component 2",
#     z="principal component 3",
#     color="class",
#     symbol="class",
#     width=800,
# )
# fig.update_layout(legend=dict(x=0, y=1))
# fig.show()

alt.Chart(df_cluster_pca).mark_circle(size=60).encode(
    
    x='principal component 1:Q',
    y='principal component 2:Q',
    color='class:N',
    shape = "class:N",
    tooltip=['CoinName', 'principal component 1', 'principal component 2', 'Algorithm']
).interactive()<jupyter_output><empty_output><jupyter_text>Table of Tradable Cryptocurrencies<jupyter_code># Table with tradable cryptos

# columns = ['CoinName', 'Algorithm', 'ProofType', 'TotalCoinSupply', 'TotalCoinsMined', 'Class']
display(df_clustered[['CoinName', 'Algorithm', 'ProofType', 'TotalCoinSupply', 'TotalCoinsMined', 'Class']])
# Print the total number of tradable cryptocurrencies
print("The number of tradable cryptocurrencies is") 
df_clustered["CoinName"].count()<jupyter_output>The number of tradable cryptocurrencies is<jupyter_text>Scatter Plot with Tradable Cryptocurrencies<jupyter_code># Scale data to create the scatter plot
df_clustered['TotalCoinsMined'] = df_clustered['TotalCoinsMined'].astype(float) / 100000000

df_clustered['TotalCoinSupply'] = df_clustered['TotalCoinSupply'].astype(float) / 100000000

# Plot the scatter with x="TotalCoinsMined" and y="TotalCoinSupply"
# df_clustered.hvplot(
#     kind="scatter", 
#     x="TotalCoinsMined", 
#     y="TotalCoinSupply", 
#     c='Class',
#     rot= 90, 
#     colormap="inferno", 
#     hover_cols=['CoinName']
# )
alt.Chart(df_clustered).mark_circle(size=60).encode(
    
    x='TotalCoinsMined',
    y='TotalCoinSupply',
    color='class:N',
    tooltip=['class', 'principal component 1', 'principal component 2', 'principal component 3']
).interactive()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11783822821376987, 'fraction_numerical': 0.011916225324987963, 'mean_word_length': 5.318631178707224, 'pattern_counts': {'":': 9, '<': 35, '<?xml version=': 0, '>': 37, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '158926', 'license': 'Unlicense', 'n_tokens_mistral': 2963, 'n_tokens_neox': 2840, 'n_words': 676, 'path': 'ClusteringCrypto Project/Starter_Files/crypto_clustering_sm.ipynb', 'repo_name': 'NJ1219/AWS-Projects', 'size': '8308'}