<jupyter_start><jupyter_text>Back at 10:35Will use a dataset about deaths in Denmark divided by week and age.Found on a website that seems very promising for getting data related to Denmark:https://www.statbank.dk/My original goal was to see if 2020 stood out in a significant way from other years using K-Means clustering.  The answer from my first investigations is "not really" but it's still good practice!<jupyter_code>import pandas as pd
import numpy as np
from sklearn.preprocessing import StandardScaler
from sklearn.cluster import KMeans
from sklearn.decomposition import PCA
from matplotlib import pyplot as plt
# %matplotlib inline # is also a good choice
# notebook better for 3d
%matplotlib notebook
df = pd.read_csv("deathsuge.csv", index_col=2,encoding= 'unicode_escape',skiprows=2)
df = df.iloc[3:,2:].T.copy()
df = df.iloc[:,:-3].copy()
# Let's scale the data using StandardScaler from sklearn
# Scaling can mean two different things: get mean to 0, get std = 1
# std = 1 makes a lot of sense if one column is dollars and another column is year
# here I think the relative sizes are important, so I won't scale the std.
# I want to use PCA eventually, and that uses vectors, 1-dimensional subspaces
# Subspaces always pass through 0, so that's why I want to change the mean so these columns
# always pass through 0.
df
scaler = StandardScaler(with_std=False)
df1 = pd.DataFrame(scaler.fit_transform(df),columns=df.columns, index = df.index)
kmeans = KMeans(n_clusters=3)
y = kmeans.fit_predict(df1)
df1["cluster"] = y
temp = list(df1[df1["cluster"] == 0].index)
n = 2
kmeans = KMeans(n_clusters=n)
y = kmeans.fit_predict(df1)
df1["cluster"] = y
for i in range(0,n):
    print("cluster = ", i)
    temp = list(df1[df1["cluster"] == i].index)
    for year in range(2007,2022):
        print(year, sum([str(year) in x for x in temp]))
    print("")
pca = PCA(n_components=2)
data = pca.fit_transform(df1.iloc[:,:-1])
data
plt.scatter(data[:,0],data[:,1])
# Back at 11:35am
# Mistake before: should have used transform rather than fit_transform after the first fit!
n = 10
kmeans = KMeans(n_clusters=n)
y = kmeans.fit_predict(df1)
df1["cluster"] = y
pca = PCA(n_components=2)
data = pca.fit_transform(df1.iloc[:,:-1])
for a,b in df1.groupby("cluster"):
    data = pca.transform(b.iloc[:,:-1])
    plt.scatter(data[:,0],data[:,1])
n = 10
kmeans = KMeans(n_clusters=n)
y = kmeans.fit_predict(df1)
df1["cluster"] = y
pca = PCA(n_components=3)
data = pca.fit_transform(df1.iloc[:,:-1])
fig = plt.figure()
ax = fig.add_subplot(111,projection='3d')
for a,b in df1.groupby("cluster"):
    data = pca.transform(b.iloc[:,:-1])
    ax.scatter(data[:,0],data[:,1],data[:,2])
fig = plt.figure()
ax = fig.add_subplot(111,projection='3d')
sum(['2011' in x for x in temp])
year
temp[3]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13259864912904373, 'fraction_numerical': 0.030572342694632066, 'mean_word_length': 5.051612903225807, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '557792', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1038, 'n_tokens_neox': 993, 'n_words': 315, 'path': 'Drafts/KU files/K-means notebook.ipynb', 'repo_name': 'ChristopherDavisUCI/UCI-Math-10-Davis', 'size': '2813'}