<jupyter_start><jupyter_text>Preparing the Data<jupyter_code>df = df.loc[df['IsTrading'] == True]
df.shape<jupyter_output><empty_output><jupyter_text>Getting rid of null values<jupyter_code>df.isnull().sum()
crypto = df.dropna()
crypto.isnull().sum()
crypto.shape
crypto.dtypes<jupyter_output><empty_output><jupyter_text>Checking for Duplicates<jupyter_code>print(f"Duplicate entries: {crypto.duplicated().sum()}")
crypto.shape
crypto = crypto.loc[crypto['TotalCoinsMined'] > 0]
crypto.shape
crypto.head(50)
crypto['TotalCoinSupply'].unique()
crypto.dtypes<jupyter_output><empty_output><jupyter_text>Making sure all features to be used are numeric<jupyter_code>crypto['TotalCoinSupply']= pd.to_numeric(crypto['TotalCoinSupply'])
crypto.head()
crypto.dtypes
crypto['Algorithm'].unique()
crypto['IsTrading'].unique()
crypto['ProofType'].unique()<jupyter_output><empty_output><jupyter_text>Drop unnecessary colums<jupyter_code>crypto = crypto.drop('CoinName', axis=1)
crypto = crypto.drop('IsTrading', axis=1)
crypto.head()
crypto = crypto.drop('Unnamed: 0', axis=1)
crypto.head()
crypto['ProofType'].unique()
crypto['Algorithm'].unique()<jupyter_output><empty_output><jupyter_text>Make Algorithm and Prooftype numeric with dummies and scale the data<jupyter_code>crypto = pd.get_dummies(crypto)
crypto.head()
scaler = StandardScaler()
scaled_data = scaler.fit_transform(crypto)
scaled_data.shape<jupyter_output><empty_output><jupyter_text>Dimensionality Reduction using PCA and further with T-sne<jupyter_code>pca = PCA(n_components=0.9)
pca_transformed_data = pca.fit_transform(scaled_data)
pca_transformed_data
pcad = pd.DataFrame(
    data=pca_transformed_data
)
pca_transformed_data.shape
pca.explained_variance_ratio_
tsne = TSNE(learning_rate=50)
transformed_features = tsne.fit_transform(pca_transformed_data)
transformed_features.shape
transformed_features
plt.scatter(transformed_features[:,0], transformed_features[:,1])
plt.show()
#There are not very clearly defined the clusters after trying with different learning rates,
#still won't give a clear distribution of clusters

dfcluster = pd.DataFrame(transformed_features)

dfcluster = dfcluster.rename(columns = {0: "Feature1", 1: "Feature2"})
dfcluster<jupyter_output><empty_output><jupyter_text>Dimensionality Reduction<jupyter_code>inertia = []
k = list(range(1, 10))

# Calculate the inertia for the range of k values
for i in k:
    km = KMeans(n_clusters=i, random_state=0)
    km.fit(transformed_features)
    inertia.append(km.inertia_)

# Create the Elbow Curve using hvPlot
elbow_data = {"k": k, "inertia": inertia}
df_elbow = pd.DataFrame(elbow_data)
df_elbow.head()
plt.plot(df_elbow['k'], df_elbow['inertia'])
plt.xticks(range(1,11))
plt.xlabel('Number of clusters')
plt.ylabel('Inertia')
plt.title('Elbow curve for customer data')
plt.show()
def get_clusters(k, data):
    # Initialize the K-Means model
    model = KMeans(n_clusters=k, random_state=0)

    # Train the model
    model.fit(data)

    # Predict clusters
    predictions = model.predict(data)

    # Create return DataFrame with predicted clusters
    data["class"] = model.labels_

    return data
clusters = get_clusters(4, dfcluster)
clusters
def show_clusters(df):
    plt.scatter(df['Feature1'], df['Feature2'], c=df['class'])
    plt.xlabel('Feature1')
    plt.ylabel('Feature2')
    plt.show()
show_clusters(clusters)<jupyter_output><empty_output><jupyter_text>Recommendation<jupyter_code>#Based on the number of clusters found we can say that our data can be clustered in 4 groups.<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1253513209668353, 'fraction_numerical': 0.008712759977515458, 'mean_word_length': 7.575903614457832, 'pattern_counts': {'":': 2, '<': 37, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '453614', 'license': 'ADSL', 'n_tokens_mistral': 1283, 'n_tokens_neox': 1228, 'n_words': 237, 'path': 'CryptocurrencyClusters.ipynb', 'repo_name': 'edipasq/unsupervised-learning-challenge', 'size': '3558'}