<jupyter_start><jupyter_text>Solar Radiation Prediction> meteorological data from the HI-SEAS weather station from four months (September through December 2016) between Mission IV and Mission V.Units:* Solar radiation: watts per meter^2* Temperature: degrees Fahrenheit* Humidity: percent* Barometric pressure: Hg* Wind direction: degrees* Wind speed: miles per hour* Sunrise/sunset: Hawaii time  Useful imports and read the data<jupyter_code>import time
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from datetime import datetime
from sklearn.model_selection import KFold
from sklearn.ensemble import RandomForestRegressor
# Read the data
df = pd.read_csv('data/SolarPrediction.csv', parse_dates=['Data'])
df.shape
# Check data format
df.head()
df.describe()<jupyter_output><empty_output><jupyter_text>Feature Engineering<jupyter_code># Convert all dates and times to unix timestamp (timezone doesn't matter now)
df['Data'] = df['Data'].dt.date.astype(str)
df['TimeSunRise'] = df['Data'] + ' ' + df['TimeSunRise']
df['TimeSunSet'] = df['Data'] + ' ' + df['TimeSunSet']
df['Data'] = df['Data'] + ' ' + df['Time']

# Convert to Unix timestamp
fields = ['Data', 'TimeSunRise', 'TimeSunSet']
for x in fields:
    df[x + '_UnixTimeStamp'] = df[x].apply(
        lambda k: int(datetime.strptime(k, "%Y-%m-%d %H:%M:%S").timestamp())
    )

# New sun time field
df['SunTime'] = df['TimeSunSet_UnixTimeStamp'] - df['TimeSunRise_UnixTimeStamp']

# Drop old columns
df.drop('UNIXTime', axis=1, inplace=True)
df.drop('Data', axis=1, inplace=True)
df.drop('Time', axis=1, inplace=True)
df.drop('TimeSunRise', axis=1, inplace=True)
df.drop('TimeSunSet', axis=1, inplace=True)

# Plot head of dataset
df.head()<jupyter_output><empty_output><jupyter_text>Visualization<jupyter_code># Plots sorted by time
df.sort_values(by=['Data_UnixTimeStamp'], ascending=[True], inplace=True)

# Radiation and Temperature for the first 2 days
f,axarr = plt.subplots(ncols=2, sharex=True)
axarr[0].set_title('Radiation for the first 2 days')
axarr[1].set_title('Temperature for the first 2 days')
sns.tsplot(data=df['Radiation'][0:600], ax=axarr[0])
sns.tsplot(data=df['Temperature'][0:600], ax=axarr[1], color="r")
plt.show()
# Plot the sun time
ax = sns.tsplot(data=df['SunTime'])
plt.title('Winter is coming')
plt.show()
# Heatmap of the dataset
heat_df = df.drop(['Data_UnixTimeStamp','TimeSunRise_UnixTimeStamp','TimeSunSet_UnixTimeStamp'], axis=1)
sns.set(context="paper", font="monospace")

# Get the correlation
corrmat = heat_df.corr()

# Set up the matplotlib figure
f, ax = plt.subplots()

# Draw the heatmap using seaborn
sns.heatmap(corrmat, vmax=.8, square=True)
plt.show()<jupyter_output><empty_output><jupyter_text>Model train<jupyter_code># Create the K-folds
k_folds = 5
kf = KFold(n_splits=k_folds, shuffle = True)

# Prepare dataset
X = df.drop(['Radiation','Data_UnixTimeStamp','TimeSunRise_UnixTimeStamp','TimeSunSet_UnixTimeStamp'] , axis=1).as_matrix()
y = df['Radiation'].as_matrix()<jupyter_output><empty_output><jupyter_text>Random Forests<jupyter_code>score = []
for train_idx, test_idx in kf.split(X):
    # Separe training and test in the Training set for k-Fold
    fold_Xtrain, fold_Xtest = X[train_idx], X[test_idx]
    fold_ytrain, fold_ytest = y[train_idx], y[test_idx]
    
    # RegrRF
    regr_rf = RandomForestRegressor(max_depth=25, random_state=2)
    regr_rf.fit(fold_Xtrain, fold_ytrain)
    
    # Test on new data
    score_kf = regr_rf.score(fold_Xtest, fold_ytest)
    score.append(score_kf)
    print("RF score=%.2f" % score_kf)
    
    # Plot results
    #pred = regr_rf.predict(fold_Xtest)
    #plt.plot(fold_ytest[0:100], color="b")
    #plt.plot(pred[0:100], color="r")
    #plt.show()
    
print("Mean score: %.2f" % np.mean(score))
print("Std score: %.2f" % np.std(score))<jupyter_output>RF score=0.86
RF score=0.85
RF score=0.87
RF score=0.86
RF score=0.85
Mean score: 0.86
Std score: 0.01<jupyter_text>LSTM<jupyter_code>#@TODO NEXT<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1439900867410161, 'fraction_numerical': 0.0161090458488228, 'mean_word_length': 6.055944055944056, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '44884', 'license': 'MIT', 'n_tokens_mistral': 1562, 'n_tokens_neox': 1507, 'n_words': 336, 'path': 'dataset-exploration/solar-predictions/solar-pred.ipynb', 'repo_name': 'Hguimaraes/kaggle-road', 'size': '4035'}