<jupyter_start><jupyter_text>perceube-se que os tipos de cluster podem estar relacionados a: condi<jupyter_code>import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt

%matplotlib inline
raw = pd.read_csv ("../datasets/2018 Merging1h.csv", header=0)
raw.head()
## Filtering variables
raw['timestamp'] = pd.to_datetime (raw['timestamp'])
raw = raw.set_index(raw.timestamp)
raw = raw.drop('timestamp', axis=1)
raw<jupyter_output><empty_output><jupyter_text>Preprocessing* Normalizo os dados* calculo a hora do dia (apenas para plotagem futura, não entra no clustering)<jupyter_code>normalized=(raw-raw.min())/(raw.max()-raw.min())
normalized
normalized['is_summer'] = normalized['is_summer']/4
normalized['is_winter'] = normalized['is_winter']/4
normalized['is_spring'] = normalized['is_spring']/4
normalized['is_fall'] = normalized['is_fall']/4
hour = raw.index.hour
from sklearn.cluster import KMeans
sse = {}
for k in range(1, 10, 1):
    model = KMeans(n_clusters=k, max_iter=1000).fit(normalized)
    #data["clusters"] = model.labels_
    #print(data["clusters"])
    sse[k] = model.inertia_ # Inertia: Sum of distances of samples to their closest cluster center
plt.figure()
plt.plot(list(sse.keys()), list(sse.values()))
plt.xlabel("Number of cluster")
plt.ylabel("SSE")
plt.show()
normalized.info()
model = KMeans(n_clusters=2)
model.fit(normalized)
f, (ax1, ax2, ax3) = plt.subplots(1, 3, sharey=True,figsize=(15,6))
ax1.scatter(hour,normalized['temp_celsius'],c=model.labels_,cmap='rainbow', alpha=0.2)
ax2.scatter(hour,normalized['p3_mean'],c=model.labels_,cmap='rainbow', alpha=0.2)
ax3.scatter(normalized['cosphi_mean'],normalized['load_factor'],c=model.labels_,cmap='rainbow', alpha=0.2)
# TODO: jitter the data to visualize 
f, (ax1, ax2, ax3) = plt.subplots(1, 3, sharey=True,figsize=(15,6))
ax1.scatter(normalized['pression'],normalized['p3_mean'],c=model.labels_,cmap='rainbow', alpha=0.2)
ax2.scatter(normalized['is_summer'],normalized['is_fall'],c=model.labels_,cmap='rainbow', alpha=0.2)
ax3.scatter(normalized['is_winter'],normalized['is_spring'],c=model.labels_,cmap='rainbow', alpha=0.2)
plt.figure(figsize=(16,8))
plt.scatter(normalized.index, normalized['p3_mean'], c=model.labels_,cmap='rainbow')
#ax1, ax2, ax3, ax4, ax5, ax6, ax7, ax8, ax9, ax10, ax11, ax12 = normalized.plot(subplots=True)
#ax1.color = model.labels_
result = pd.DataFrame(columns=raw[model.labels_ == 0].mean().index)
for k in range(0, 4):
    result.loc[k] = raw[model.labels_ == k].mean().values
result
for k in range(0, 4):
    print (raw[model.labels_ == k]['temp_celsius'].count())<jupyter_output><empty_output><jupyter_text>PCA<jupyter_code>from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
scaler.fit(raw.drop('p3_mean', axis=1))
scaled_data = scaler.transform(raw.drop('p3_mean', axis=1))
from sklearn.decomposition import PCA
pca = PCA(n_components=2)
pca.fit(scaled_data)

x_pca = pca.transform(scaled_data)
plt.figure(figsize=(8,6))
plt.scatter(x_pca[:,0],x_pca[:,1],c=raw['p3_mean'],cmap='plasma', alpha=0.1)
plt.xlabel('First principal component')
plt.ylabel('Second Principal Component')
# Interretando a composição dos componentes
df_comp = pd.DataFrame(pca.components_,columns=raw.drop('p3_mean', axis=1).columns)
plt.figure(figsize=(12,6))
sns.heatmap(df_comp,cmap='plasma')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17105652559928974, 'fraction_numerical': 0.029002663509914175, 'mean_word_length': 7.9655172413793105, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '391346', 'license': 'MIT', 'n_tokens_mistral': 1344, 'n_tokens_neox': 1292, 'n_words': 223, 'path': 'artificial_intelligence/06 - Daily consumption clustering/.ipynb_checkpoints/daily clustering 2 (whole 2018, with weather)-checkpoint.ipynb', 'repo_name': 'LeonardoSanBenitez/LorisWeb', 'size': '3379'}