<jupyter_start><jupyter_text>PCA on the meteorological elements to understand the tempfile:///C:/Users/Shelly/Downloads/sustainability-11-06740-v2.pdf<jupyter_code>from sklearn.preprocessing import StandardScaler

features = ['temp','temp_min','temp_max','pressure','humidity','wind_speed', 'wind_deg','hour','Day','Night']

# Separating out the features
x = Givatayim_df.loc[:, features].values

# Separating out the target
y = Givatayim_df.loc[:,['dt']].values

# Standardizing the features
z = StandardScaler().fit_transform(x)
from sklearn.decomposition import PCA

pca = PCA(n_components=2)

principalComponents = pca.fit_transform(z)

principalDf = pd.DataFrame(data = principalComponents
             , columns = ['principal component 1', 'principal component 2'])
principalDf.head()
finalDf = pd.concat([principalDf, Givatayim_df[['day_night']]], axis = 1)
finalDf.head()
# we went from 4 features down to 2 features.
# how much information have we lost?
# how much information is captured by each principal component?
print(pca.explained_variance_ratio_)
print(sum(pca.explained_variance_ratio_))
from matplotlib import pyplot as plt

fig = plt.figure(figsize = (8,8))
ax = fig.add_subplot(1,1,1) 
ax.set_xlabel('Principal Component 1', fontsize = 15)
ax.set_ylabel('Principal Component 2', fontsize = 15)
ax.set_title('2 component PCA', fontsize = 20)

targets = ['Day', 'Night']
colors = ['r', 'g', 'b']
for target, color in zip(targets,colors):
    indicesToKeep = finalDf['day_night'] == target
    ax.scatter(finalDf.loc[indicesToKeep, 'principal component 1']
               , finalDf.loc[indicesToKeep, 'principal component 2']
               , c = color
               , s = 50)
ax.legend(targets)
ax.grid()

finalDf = pd.concat([principalDf, Givatayim_df[['Date']]], axis = 1)
finalDf.head()
from matplotlib import pyplot as plt

fig = plt.figure(figsize = (8,8))
ax = fig.add_subplot(1,1,1) 
ax.set_xlabel('Principal Component 1', fontsize = 15)
ax.set_ylabel('Principal Component 2', fontsize = 15)
ax.set_title('2 component PCA', fontsize = 20)

targets = ['2020-10-08','2020-10-09','2020-10-10','2020-10-11','2020-10-12','2020-10-13','2020-10-14','2020-10-15','2020-10-16','2020-10-17','2020-10-18','2020-10-19','2020-10-20','2020-10-21','2020-10-22','2020-10-23','2020-10-24','2020-10-25','2020-10-26','2020-10-27','2020-10-28','2020-10-29','2020-10-30','2020-10-31','2020-11-01','2020-11-02','2020-11-03','2020-11-04','2020-11-05','2020-11-06','2020-11-07','2020-11-08','2020-11-09','2020-11-10','2020-11-11','2020-11-12','2020-11-13','2020-11-14','2020-11-15']
colors = ['g','b','c','k','tab:blue','tab:orange','r','y','tab:pink','tab:red','m','tab:brown','goldenrod','lightcoral','salmon','gold','hotpink','violet','blue','navy','indigo','teal','royalblue','forestgreen','greenyellow','tomato','peru','aquamarine','azure','lightblue','palevioletred','lavender','wheat','lime','khaki','chocolate']
for target, color in zip(targets,colors):
    indicesToKeep = finalDf['Date'] == target
    ax.scatter(finalDf.loc[indicesToKeep, 'principal component 1']
               , finalDf.loc[indicesToKeep, 'principal component 2']
               , c = color
               , s = 50)
ax.legend(targets)
ax.grid()

finalDf = pd.concat([principalDf, Givatayim_df[['hour']]], axis = 1)
finalDf.head()
from matplotlib import pyplot as plt

fig = plt.figure(figsize = (8,8))
ax = fig.add_subplot(1,1,1) 
ax.set_xlabel('Principal Component 1', fontsize = 15)
ax.set_ylabel('Principal Component 2', fontsize = 15)
ax.set_title('2 component PCA by hour', fontsize = 20)

targets = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23]
# warm colors represent day time, cold colors represent night time
colors = ['g','b','c','k','tab:blue','tab:orange','r','y','tab:pink','tab:red','m','tab:brown','goldenrod','lightcoral','salmon','gold','hotpink','violet','blue','navy','indigo','teal','royalblue','forestgreen']
for target, color in zip(targets,colors):
    indicesToKeep = finalDf['hour'] == target
    ax.scatter(finalDf.loc[indicesToKeep, 'principal component 1']
               , finalDf.loc[indicesToKeep, 'principal component 2']
               , c = color
               , s = 50)
ax.legend(targets)
ax.grid()

finalDf = pd.concat([principalDf, Givatayim_df[['Month']]], axis = 1)
finalDf.head()
from matplotlib import pyplot as plt

fig = plt.figure(figsize = (8,8))
ax = fig.add_subplot(1,1,1) 
ax.set_xlabel('Principal Component 1', fontsize = 15)
ax.set_ylabel('Principal Component 2', fontsize = 15)
ax.set_title('2 component PCA by month', fontsize = 20)

targets = [10,11]
colors = ['r', 'g', 'b']
for target, color in zip(targets,colors):
    indicesToKeep = finalDf['Month'] == target
    ax.scatter(finalDf.loc[indicesToKeep, 'principal component 1']
               , finalDf.loc[indicesToKeep, 'principal component 2']
               , c = color
               , s = 50)
ax.legend(targets)
ax.grid()

from sklearn.preprocessing import StandardScaler

features = ['temp_min','temp_max','pressure','humidity','wind_speed', 'wind_deg','hour','Day','Night']

# Separating out the features
x = Givatayim_df.loc[:, features].values

# Separating out the target
y = Givatayim_df.loc[:,['temp']].values

# Standardizing the features
z = StandardScaler().fit_transform(x)
from sklearn.decomposition import PCA

pca = PCA(n_components=2)

principalComponents = pca.fit_transform(z)

principalDf = pd.DataFrame(data = principalComponents
             , columns = ['principal component 1', 'principal component 2'])
finalDf = pd.concat([principalDf, Givatayim_df[['temp']]], axis = 1)
finalDf.head()
from matplotlib import pyplot as plt
import numpy as np 

fig = plt.figure(figsize = (8,8))
ax = fig.add_subplot(1,1,1) 
ax.set_xlabel('Principal Component 1', fontsize = 15)
ax.set_ylabel('Principal Component 2', fontsize = 15)
ax.set_title('2 component PCA', fontsize = 20)

targets = list(np.arange(18,40,0.3))
colors = ['g','b','c','k','tab:blue','tab:orange','r','y','tab:pink','tab:red','m','tab:brown','goldenrod','lightcoral','salmon','gold','hotpink','violet','blue','navy','indigo','teal','royalblue','forestgreen']
for target, color in zip(targets,colors):
    indicesToKeep = finalDf['temp'] == target
    ax.scatter(finalDf.loc[indicesToKeep, 'principal component 1']
               , finalDf.loc[indicesToKeep, 'principal component 2']
               , c = color
               , s = 50)
ax.legend(targets)
ax.grid()<jupyter_output><empty_output><jupyter_text>split the data to train and test<jupyter_code># according to class
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from scipy.stats.stats import pearsonr
import warnings
import math
import random
from sklearn.decomposition import PCA

X = Givatayim_df[['temp_min','temp_max','humidity','pressure','wind_speed','wind_deg']].values
Y = Givatayim_df['temp'].values
n = X.shape[0]
tr_size = math.floor(0.8*n)
te_size = n-tr_size
tr_ind = random.sample(range(n),tr_size)
Xtr = X[tr_ind,]
Xte = np.delete(X,tr_ind,0)
Ytr = Y[tr_ind]
Yte = np.delete(Y,tr_ind)

print(n,Xtr.shape, Xte.shape, Ytr.shape, Yte.shape)
# acoording to: https://towardsdatascience.com/methods-for-dealing-with-imbalanced-data-5b761be45a18
# after dividing to train and test i can do oversampling of the imbalanced data (dates with less samples per hour)
# i shouldn't delete (downsampling) because i dont have enough data anyway

from sklearn.utils import resample

# Separate input features and target
y = Givatayim_df.temp
X = Givatayim_df.drop('temp', axis=1)

# setting up testing and training sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=4)

# concatenate our training data back together
X = pd.concat([X_train, y_train], axis=1)

# separate minority and majority classes
majority = X[X.Class==0]
fraud = X[X.Class==1]

# upsample minority
fraud_upsampled = resample(fraud,
                          replace=True, # sample with replacement
                          n_samples=len(not_fraud), # match number in majority class
                          random_state=27) # reproducible results

# combine majority and upsampled minority
upsampled = pd.concat([not_fraud, fraud_upsampled])

# check new class counts
upsampled.Class.value_counts()
    1    213245
    0    213245<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16541173658419983, 'fraction_numerical': 0.06047567825983029, 'mean_word_length': 4.998566308243728, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '115554', 'license': 'MIT', 'n_tokens_mistral': 3310, 'n_tokens_neox': 2925, 'n_words': 612, 'path': 'Givatayim_weather_data_science.ipynb', 'repo_name': 'shellyalmo/weather_data_analysis', 'size': '8367'}