<jupyter_start><jupyter_text>Import Data<jupyter_code>import numpy as np # linear algebra
import pandas as pd 
import seaborn as sns
import matplotlib.pyplot as plt
import regex as re
from datetime import datetime
% matplotlib inline



df = pd.read_csv("/content/TRAIN.csv")
df.head()
!wget -O "solar_radiation_prediction_ai_challenge-dataset.zip" "https://dockship-job-models.s3.ap-south-1.amazonaws.com/a379f6594a948741d92dba42178aff0b?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=<PASSWORD>%2F<PASSWORD>%2<PASSWORD>%2F<PASSWORD>&X-Amz-Date=20201220T060236Z&X-Amz-Expires=1800&X-Amz-Signature=72cf6270ff15749ab0b4ee1b58e52ec4206d98e808131112dd4ab65d92498829&X-Amz-SignedHeaders=host&response-content-disposition=attachment%3B%20filename%3D%22solar_radiation_prediction_ai_challenge-dataset.zip%22"
#!unzip "solar_radiation_prediction_ai_challenge-dataset.zip"<jupyter_output><empty_output><jupyter_text>Investigate Existing Correlations<jupyter_code>corrmat = df.corr()
sns.heatmap(corrmat, vmax=.8, square=True)
#drop high radiation values
df = df[df['Radiation'] <= 1200]
df.Radiation.skew()
from scipy.stats import skew,kurtosis,zscore
def distribution_check(df,Cols):
    plt.figure(figsize=(12,7))
    fig = 1
    i = (len(Cols)//3)+1
    for col in Cols:
        sk = " Skewness : " + str(skew(df[col])) +"\nKurtosis :" + str(kurtosis(df[col]))
        plt.subplot(i, 3, fig)
        sns.distplot(df[col]).set_title(sk)
        fig = fig+1
distribution_check(df,['Radiation'])
df['Radiation'] = np.log(df['Radiation'])
y = df['Radiation']
df.Radiation.skew()<jupyter_output><empty_output><jupyter_text>Feature Engineering<jupyter_code>#Covert time to_datetime
#Add column 'hour'
df['Time_conv'] =  pd.to_datetime(df['Time'], format='%H:%M:%S')
df['hour'] = pd.to_datetime(df['Time_conv'], format='%H:%M:%S').dt.hour

#Add column 'month'
df['month'] = pd.to_datetime(df['UNIXTime'].astype(int), unit='s').dt.month

df['weekday'] = pd.to_datetime(df['UNIXTime'].astype(int), unit='s').dt.weekday

df['week'] = pd.to_datetime(df['UNIXTime'].astype(int), unit='s').dt.week

df['year'] = pd.to_datetime(df['UNIXTime'].astype(int), unit='s').dt.year

#Duration of Day
df['total_time'] = pd.to_datetime(df['TimeSunSet'], format='%H:%M:%S').dt.hour - pd.to_datetime(df['TimeSunRise'], format='%H:%M:%S').dt.hour
df.head()<jupyter_output>/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:11: FutureWarning: Series.dt.weekofyear and Series.dt.week have been deprecated.  Please use Series.dt.isocalendar().week instead.
  # This is added back by InteractiveShellApp.init_path()<jupyter_text>Data Visualization<jupyter_code>ax = plt.axes()
sns.barplot(x="hour", y='Radiation', data=df, palette="BuPu", ax = ax)
ax.set_title('Mean Radiation by Hour')
plt.show()
ax = plt.axes()
sns.barplot(x="month", y='Radiation', data=df, palette="BuPu", ax = ax, order=[9,10,11,12,1])
ax.set_title('Mean Radiation by Month')
plt.show()
ax = plt.axes()
sns.barplot(x="hour", y='Humidity', data=df, palette=("coolwarm"), ax = ax)
ax.set_title('Humidity by Hour')
plt.show()
ax = plt.axes()
sns.barplot(x="total_time", y='Radiation', data=df, palette="BuPu", ax = ax)
ax.set_title('Radiation by Total Daylight Hours')
plt.show()

df['Radiation'].hist()
df['Radiation'].skew()<jupyter_output><empty_output><jupyter_text>Predict using Linear ModelCurrent caveats include the deletion of radiation data equal to or below 10 units. The MSE is still really high. I'm looking for ways to drop this.<jupyter_code>df = df.drop([ 'idx','Data', 'Time', 'TimeSunRise', 'TimeSunSet','Time_conv','Pressure','Humidity','WindDirection(Degrees)','Speed','hour'], axis=1)
df
dummies = pd.get_dummies(df['month'],drop_first = True)
dummies1 = pd.get_dummies(df['year'],drop_first = True)
df = df.drop(['month','year'],1)
df = pd.concat([df, dummies,dummies1], 1)
df
from sklearn.preprocessing import MinMaxScaler
scaler = MinMaxScaler()
y = df['Radiation']
X = df.drop(['Radiation'], axis=1)
X = scaler.fit_transform(X)
X

from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.05, random_state = 42)
import xgboost as xg
from sklearn.metrics import mean_squared_error as MSE 
xgb_r = xg.XGBRegressor( n_estimators = 100, max_depth = 25, learning_rate = 0.1) 
   
xgb_r.fit(X_train, y_train) 
   
pred = xgb_r.predict(X_test) 
  
rmse = np.sqrt(MSE(y_test, pred)) 
print("RMSE : % f" %(rmse)) 
from sklearn.metrics import mean_squared_error as MSE, r2_score
r2_score(y_test,pred)*100
pred
from sklearn.ensemble import RandomForestRegressor
regr = RandomForestRegressor(max_depth = 50, random_state=0,n_estimators = 1000,verbose = 2)
regr.fit(X_train, y_train)
pred1 = regr.predict(X_test) 
from sklearn.metrics import mean_squared_error as MSE
rmse = np.sqrt(MSE(y_test, pred1)) 
print("RMSE : % f" %(rmse)) 
plt.scatter(y_test,pred1)
plt.xlabel('Y Test')
plt.ylabel('Predicted Y')
plt.show()
#from yellowbrick.datasets import load_concrete
#from yellowbrick.regressor import ResidualsPlot
#visualizer = ResidualsPlot(xgb_r, hist=False, qqplot=True)

#visualizer.fit(X_train, y_train)  
#visualizer.score(X_test, y_test)<jupyter_output>[18:36:58] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.<jupyter_text>Feature selection<jupyter_code>from mlxtend.feature_selection import SequentialFeatureSelector as SFS
from sklearn.ensemble import RandomForestRegressor
sfs1 = SFS(RandomForestRegressor(), 
           k_features=10, 
           forward=False, 
           floating=False, 
           verbose = 2,
           scoring='neg_root_mean_squared_error',
           cv=3)

sfs1 = sfs1.fit(np.array(X_train), y_train)
sfs1.k_feature_idx_
X_train.columns[list(sfs1.k_feature_idx_)]<jupyter_output><empty_output><jupyter_text>**Fitting**<jupyter_code>df = pd.read_csv("/content/TEST.csv")

df['Time_conv'] =  pd.to_datetime(df['Time'], format='%H:%M:%S')
df['hour'] = pd.to_datetime(df['Time_conv'], format='%H:%M:%S').dt.hour

#Add column 'month'
df['month'] = pd.to_datetime(df['UNIXTime'].astype(int), unit='s').dt.month

df['weekday'] = pd.to_datetime(df['UNIXTime'].astype(int), unit='s').dt.weekday

df['week'] = pd.to_datetime(df['UNIXTime'].astype(int), unit='s').dt.week

#Add column 'year'
df['year'] = pd.to_datetime(df['UNIXTime'].astype(int), unit='s').dt.year



#Duration of Day
df['total_time'] = pd.to_datetime(df['TimeSunSet'], format='%H:%M:%S').dt.hour - pd.to_datetime(df['TimeSunRise'], format='%H:%M:%S').dt.hour
dummies = pd.get_dummies(df['month'],df['year'])
df = df.drop(['month','year'],1)
df = pd.concat([df, dummies], 1)
df.head()
df = df.reindex(columns = X_train.columns, fill_value=0)
df
X = df

X = scaler.transform(X)
X
y_pred  = regr.predict(X)
y_pred
y_pred = np.where(y_pred < 0, 1, y_pred)
y_pred
y_pred = np.exp(y_pred)
y_pred
output = pd.DataFrame(y_pred)
output.to_csv('output.csv', index = False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16678576532799771, 'fraction_numerical': 0.026296984421895098, 'mean_word_length': 6.428874734607219, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 31, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '452605', 'license': 'MIT', 'n_tokens_mistral': 2903, 'n_tokens_neox': 2734, 'n_words': 456, 'path': 'Solar_Radiation-Dockship/solar-radiation.ipynb', 'repo_name': 'kushagra-18/Machine-Learning-Hackethons', 'size': '7013'}