<jupyter_start><jupyter_text>Importing required libraries<jupyter_code>import pandas as pd
import numpy as np
from sklearn import metrics<jupyter_output><empty_output><jupyter_text>Checking Linearity with Past Drought Values  Initializing Feature and Response matrices stored as .CSV files earlier  Training Dataset<jupyter_code>X_train = np.loadtxt(r'R:\Ryerson\Misc\Datasets\Preprocessed Data Files\New + pastDroughtValues\30_Day_Window_Scaled + pastDroughtValues\X_train_30_day_window_scaled_+_pastDroughtValues.csv', delimiter=',')

y_target_train = np.loadtxt(r'R:\Ryerson\Misc\Datasets\Preprocessed Data Files\New + pastDroughtValues\30_Day_Window_Scaled + pastDroughtValues\y_target_train_30_day_window.csv', delimiter=',')
X_train.shape

y_target_train.shape<jupyter_output><empty_output><jupyter_text>Validation Dataset<jupyter_code>X_valid = np.loadtxt(r'R:\Ryerson\Misc\Datasets\Preprocessed Data Files\New + pastDroughtValues\30_Day_Window_Scaled + pastDroughtValues\X_valid_30_day_window_scaled_+_pastDroughtValues.csv', delimiter=',')

y_target_valid = np.loadtxt(r'R:\Ryerson\Misc\Datasets\Preprocessed Data Files\New + pastDroughtValues\30_Day_Window_Scaled + pastDroughtValues\y_target_valid_30_day_window.csv', delimiter=',')
X_valid.shape

y_target_valid.shape<jupyter_output><empty_output><jupyter_text>Testing Dataset<jupyter_code>X_test = np.loadtxt(r'R:\Ryerson\Misc\Datasets\Preprocessed Data Files\New + pastDroughtValues\30_Day_Window_Scaled + pastDroughtValues\X_test_30_day_window_scaled_+_pastDroughtValues.csv', delimiter=',')

y_target_test = np.loadtxt(r'R:\Ryerson\Misc\Datasets\Preprocessed Data Files\New + pastDroughtValues\30_Day_Window_Scaled + pastDroughtValues\y_target_test_30_day_window.csv', delimiter=',')
X_test.shape

y_target_test.shape<jupyter_output><empty_output><jupyter_text>_Concatenating X_valid & X_test into a single test set_<jupyter_code>X_Test = np.concatenate((X_valid, X_test), axis=0)
X_Test.shape<jupyter_output><empty_output><jupyter_text>_Concatenating y_target_valid & y_target_test into a single array_<jupyter_code>y_Target = np.concatenate((y_target_valid, y_target_test), axis=0)
y_Target.shape<jupyter_output><empty_output><jupyter_text>Training Linear Regression on this dataset and observing the R_2 Score<jupyter_code>from sklearn.linear_model import LinearRegression
linearRegressor = LinearRegression().fit(X_train, y_target_train)
# Checking the accuracy
from sklearn.metrics import r2_score
yPredicted = linearRegressor.predict(X_Test)
print(r2_score(y_Target, yPredicted))<jupyter_output>0.6020546834278234<jupyter_text>Checking Linearity with just the 18 Features before Feature Selection  Initializing Feature and Response matrices stored as .CSV files earlier  Training Dataset<jupyter_code>X_train = np.loadtxt(r'R:\Ryerson\Misc\Datasets\Preprocessed Data Files\Old + Probably Wrong\30_Day_Window_Scaled\X_train_30_day_window_scaled.csv', delimiter=',')

y_target_train = np.loadtxt(r'R:\Ryerson\Misc\Datasets\Preprocessed Data Files\Old + Probably Wrong\30_Day_Window_Scaled\y_target_train_30_day_window_scaled.csv', delimiter=',')
X_train.shape

y_target_train.shape<jupyter_output><empty_output><jupyter_text>Validation Dataset<jupyter_code>X_valid = np.loadtxt(r'R:\Ryerson\Misc\Datasets\Preprocessed Data Files\Old + Probably Wrong\30_Day_Window_Scaled\X_valid_30_day_window_scaled.csv', delimiter=',')

y_target_valid = np.loadtxt(r'R:\Ryerson\Misc\Datasets\Preprocessed Data Files\Old + Probably Wrong\30_Day_Window_Scaled\y_target_valid_30_day_window_scaled.csv', delimiter=',')
X_valid.shape

y_target_valid.shape<jupyter_output><empty_output><jupyter_text>Testing Dataset<jupyter_code>X_test = np.loadtxt(r'R:\Ryerson\Misc\Datasets\Preprocessed Data Files\Old + Probably Wrong\30_Day_Window_Scaled\X_test_30_day_window_scaled.csv', delimiter=',')

y_target_test = np.loadtxt(r'R:\Ryerson\Misc\Datasets\Preprocessed Data Files\Old + Probably Wrong\30_Day_Window_Scaled\y_target_test_30_day_window_scaled.csv', delimiter=',')
X_test.shape

y_target_test.shape<jupyter_output><empty_output><jupyter_text>_Concatenating X_valid & X_test into a single test set_<jupyter_code>X_Test = np.concatenate((X_valid, X_test), axis=0)
X_Test.shape<jupyter_output><empty_output><jupyter_text>_Concatenating y_target_valid & y_target_test into a single array_<jupyter_code>y_Target = np.concatenate((y_target_valid, y_target_test), axis=0)
y_Target.shape<jupyter_output><empty_output><jupyter_text>Training Linear Regression on this dataset and observing the R_2 Score<jupyter_code>from sklearn.linear_model import LinearRegression
linearRegressor = LinearRegression().fit(X_train, y_target_train)
# Checking the accuracy
from sklearn.metrics import r2_score
yPredicted = linearRegressor.predict(X_Test)
print(r2_score(y_Target, yPredicted))<jupyter_output>0.15287767976987376<jupyter_text>Checking Linearity with just the 8 Features after Feature Selection  Initializing Feature and Response matrices stored as .CSV files earlier  Training Dataset<jupyter_code>X_train = np.loadtxt(r'R:\Ryerson\Misc\Datasets\Preprocessed Data Files\NEW + FEATURE SELECTION\30_Day_Window_Scaled + FeatureSelection\X_train_30_day_window_scaled_+_FeatureSelection.csv', delimiter=',')

y_target_train = np.loadtxt(r'R:\Ryerson\Misc\Datasets\Preprocessed Data Files\NEW + FEATURE SELECTION\30_Day_Window_Scaled + FeatureSelection\y_target_train_30_day_window.csv', delimiter=',')
X_train.shape

y_target_train.shape<jupyter_output><empty_output><jupyter_text>Validation Dataset<jupyter_code>X_valid = np.loadtxt(r'R:\Ryerson\Misc\Datasets\Preprocessed Data Files\NEW + FEATURE SELECTION\30_Day_Window_Scaled + FeatureSelection\X_valid_30_day_window_scaled_+_FeatureSelection.csv', delimiter=',')

y_target_valid = np.loadtxt(r'R:\Ryerson\Misc\Datasets\Preprocessed Data Files\NEW + FEATURE SELECTION\30_Day_Window_Scaled + FeatureSelection\y_target_valid_30_day_window.csv', delimiter=',')
X_valid.shape

y_target_valid.shape<jupyter_output><empty_output><jupyter_text>Testing Dataset<jupyter_code>X_test = np.loadtxt(r'R:\Ryerson\Misc\Datasets\Preprocessed Data Files\NEW + FEATURE SELECTION\30_Day_Window_Scaled + FeatureSelection\X_test_30_day_window_scaled_+_FeatureSelection.csv', delimiter=',')

y_target_test = np.loadtxt(r'R:\Ryerson\Misc\Datasets\Preprocessed Data Files\NEW + FEATURE SELECTION\30_Day_Window_Scaled + FeatureSelection\y_target_test_30_day_window.csv', delimiter=',')
X_test.shape

y_target_test.shape<jupyter_output><empty_output><jupyter_text>_Concatenating X_valid & X_test into a single test set_<jupyter_code>X_Test = np.concatenate((X_valid, X_test), axis=0)
X_Test.shape<jupyter_output><empty_output><jupyter_text>_Concatenating y_target_valid & y_target_test into a single array_<jupyter_code>y_Target = np.concatenate((y_target_valid, y_target_test), axis=0)
y_Target.shape<jupyter_output><empty_output><jupyter_text>Training Linear Regression on this dataset and observing the R_2 Score<jupyter_code>from sklearn.linear_model import LinearRegression
linearRegressor = LinearRegression().fit(X_train, y_target_train)
# Checking the accuracy
from sklearn.metrics import r2_score
yPredicted = linearRegressor.predict(X_Test)
print(r2_score(y_Target, yPredicted))<jupyter_output>0.13475902513077265
{'fraction_non_alphanumeric': 0.09482640318375189, 'fraction_numerical': 0.01962398792369974, 'mean_word_length': 9.926536731634183, 'pattern_counts': {'":': 0, '<': 74, '<?xml version=': 0, '>': 74, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '19', 'dir': 'jupyter-structured-clean-dedup', 'id': '134232', 'license': 'CC0-1.0', 'n_tokens_mistral': 2755, 'n_tokens_neox': 2655, 'n_words': 364, 'path': 'Check_Linearity_of_Dataset.ipynb', 'repo_name': 'bhupindersingh47/MRP-2021', 'size': '7287'}