<jupyter_start><jupyter_text>Ask Question Can we predict the maximum temperature tomorrow for Seattle, Washington given one year of historical data?  The data we have available is the maximum temperatures in Seattle from the National Oceanic and Atmospheric Administration's (NOAA) [Climate Data Online Tool](https://www.ncdc.noaa.gov/cdo-web/).  Data Acqusition<jupyter_code># Pandas is used for data manipulation
import pandas as pd
from matplotlib import pyplot as plt
import joblib

# Read in data as pandas dataframe and display first 5 rows

features_train = pd.read_csv('data/train.csv')
features_val = pd.read_csv('data/val.csv')
#features = pd.concat([data_train, data_val])


from sklearn.preprocessing import LabelEncoder<jupyter_output><empty_output><jupyter_text>Identify Anomalies<jupyter_code>print('The shape of our features is:', features_train.shape)
print('The shape of our features is:', features_val.shape)

# Descriptive statistics for each column
features_train.describe()
""""
# Use datetime for dealing with dates
import datetime

# Get years, months, and days
years = features['ANIO']
months = features['MES']
days = features['DIA']

# List and then convert to datetime object
dates = [str(int(year)) + '-' + str(int(month)) + '-' + str(int(day)) for year, month, day in zip(years, months, days)]
dates = [datetime.datetime.strptime(date, '%Y-%m-%d') for date in dates]
"""
features_train['DIA_SEMANA']=pd.to_datetime(features_train['ANIO']*10000+features_train['MES']*100+features_train['DIA'],format='%Y%m%d').dt.dayofweek
features_val['DIA_SEMANA']=pd.to_datetime(features_val['ANIO']*10000+features_val['MES']*100+features_val['DIA'],format='%Y%m%d').dt.dayofweek
features_val
features_train
"""
# Set up the plotting layout
fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(nrows=2, ncols=2, figsize = (10,10))
fig.autofmt_xdate(rotation = 45)

# Actual max temperature measurement
ax1.plot(dates, features['actual'])
ax1.set_xlabel(''); ax1.set_ylabel('Temperature'); ax1.set_title('Max Temp')

# Temperature from 1 day ago
ax2.plot(dates, features['temp_1'])
ax2.set_xlabel(''); ax2.set_ylabel('Temperature'); ax2.set_title('Previous Max Temp')

# Temperature from 2 days ago
ax3.plot(dates, features['temp_2'])
ax3.set_xlabel('Date'); ax3.set_ylabel('Temperature'); ax3.set_title('Two Days Prior Max Temp')

# Friend Estimate
ax4.plot(dates, features['friend'])
ax4.set_xlabel('Date'); ax4.set_ylabel('Temperature'); ax4.set_title('Friend Estimate')

plt.tight_layout(pad=2)
"""<jupyter_output><empty_output><jupyter_text>Data Preparation  One-Hot Encoding<jupyter_code># One-hot encode categorical features


"""
le = LabelEncoder()
le.fit(features['CLASE'].unique())
features['CLASE']=le.transform(features['BARRIO'])
"""
features_train = pd.get_dummies(features_train)
features_train=features_train.groupby(['DIA_SEMANA','MES','ANIO','ESPECIAL','CLUSTER','SEMANA']).agg({'CLASE_atropello':'sum','CLASE_caida ocupante':'sum','CLASE_choque':'sum','CLASE_otro':'sum','CLASE_volcamiento':'sum'})
nombre_columnasTipo=['atropello', 'caidaocupante','choque','otro','volcamiento']
features_train.columns = nombre_columnasTipo
features_train = features_train.reset_index()

features_train.loc[features_train['CLUSTER'] == 0, 'CLUSTER'] = 'cluster0'
features_train.loc[features_train['CLUSTER'] == 1, 'CLUSTER'] = 'cluster1'
features_train.loc[features_train['CLUSTER'] == 2, 'CLUSTER'] = 'cluster2'

features_train = pd.get_dummies(features_train)


features_val = pd.get_dummies(features_val)
features_val=features_val.groupby(['DIA_SEMANA','MES','ANIO','ESPECIAL','CLUSTER','SEMANA']).agg({'CLASE_atropello':'sum','CLASE_caida ocupante':'sum','CLASE_choque':'sum','CLASE_otro':'sum','CLASE_volcamiento':'sum'})
nombre_columnasTipo=['atropello', 'caidaocupante','choque','otro','volcamiento']
features_val.columns = nombre_columnasTipo
features_val = features_val.reset_index()

features_val.loc[features_val['CLUSTER'] == 0, 'CLUSTER'] = 'cluster0'
features_val.loc[features_val['CLUSTER'] == 1, 'CLUSTER'] = 'cluster1'
features_val.loc[features_val['CLUSTER'] == 2, 'CLUSTER'] = 'cluster2'

features_val = pd.get_dummies(features_val)


print('Shape of features after one-hot encoding:', features_val.shape)
print('Shape of train after one-hot encoding:', features_train.shape)<jupyter_output>Shape of features after one-hot encoding: (1089, 13)
Shape of train after one-hot encoding: (4371, 13)<jupyter_text>Features and Labels and Convert Data to Arrays<jupyter_code># Use numpy to convert to arrays
import numpy as np

# Labels are the values we want to predict

label_train = np.array(features_train[['atropello', 'caidaocupante','choque','otro','volcamiento']])
label_val = np.array(features_val[['atropello', 'caidaocupante','choque','otro','volcamiento']])

# Remove the labels from the features
# axis 1 refers to the columns
features_train= features_train.drop(['atropello', 'caidaocupante','choque','otro','volcamiento'], axis = 1)
features_val= features_val.drop(['atropello', 'caidaocupante','choque','otro','volcamiento'], axis = 1)

feature_list = list(features_train.columns)

features_train= np.array(features_train)
features_val= np.array(features_val)<jupyter_output><empty_output><jupyter_text>Conjuntos de prueba<jupyter_code>print('Training Features Shape:', features_train.shape)
print('Training Labels Shape:', label_train.shape)
print('Testing Features Shape:', features_val.shape)
print('Testing Labels Shape:', label_val.shape)<jupyter_output>Training Features Shape: (4371, 8)
Training Labels Shape: (4371, 5)
Testing Features Shape: (1089, 8)
Testing Labels Shape: (1089, 5)<jupyter_text>Establish Baseline<jupyter_code>"""
# The baseline predictions are the historical averages
baseline_preds = test_features[:, feature_list.index('average')]

# Baseline errors, and display average baseline error
baseline_errors = abs(baseline_preds - test_labels)
print('Average baseline error: ', round(np.mean(baseline_errors), 2), 'degrees.')
"""<jupyter_output><empty_output><jupyter_text>Train Model<jupyter_code># Import the model we are using
from sklearn.ensemble import RandomForestRegressor

# Instantiate model 
rf = RandomForestRegressor(n_estimators= 1000, min_samples_leaf=50, random_state=42)

# Train the model on training data
rf.fit(features_train,label_train);<jupyter_output><empty_output><jupyter_text>Make Predictions on Test Data<jupyter_code>from sklearn.metrics import mean_squared_error as rmse
# Use the forest's predict method on the test data
prediccion_validacion = rf.predict(features_val)
prediccion_entrenamiento=rf.predict(features_train)
# Calculate the absolute errors
error=rmse(label_val, prediccion_validacion)
error2=rmse(label_train, prediccion_entrenamiento)

# Print out the mean absolute error (mae)
print('error validacion:', error,"erro prueva",error2)
#plt.plot(prediccion_validacion[80])
#plt.plot(test_labels[80])
plt.plot(prediccion_entrenamiento[100:500,3])

plt.plot(label_train[100:500,3])


# Get numerical feature importances
importances = list(rf.feature_importances_)

# List of tuples with variable and importance
feature_importances = [(feature, round(importance, 2)) for feature, importance in zip(feature_list, importances)]

# Sort the feature importances by most important first
feature_importances = sorted(feature_importances, key = lambda x: x[1], reverse = True)
joblib.dump(rf, 'models/arbol_dia.pkl')
# Print out the feature and importances 
[print('Variable: {:20} Importance: {}'.format(*pair)) for pair in feature_importances];<jupyter_output>Variable: CLUSTER_cluster1     Importance: 0.81
Variable: DIA_SEMANA           Importance: 0.09
Variable: ESPECIAL             Importance: 0.04
Variable: CLUSTER_cluster0     Importance: 0.02
Variable: CLUSTER_cluster2     Importance: 0.02
Variable: MES                  Importance: 0.01
Variable: SEMANA               Importance: 0.01
Variable: ANIO                 Importance: 0.0
{'fraction_non_alphanumeric': 0.13529856386999245, 'fraction_numerical': 0.019652305366591082, 'mean_word_length': 7.801552106430155, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 30, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '19880', 'license': 'Apache-2.0', 'n_tokens_mistral': 2797, 'n_tokens_neox': 2639, 'n_words': 580, 'path': 'directorio de trabajo/Jorge/no puedo creer que esto funcione/.ipynb_checkpoints/arbol_dia-checkpoint.ipynb', 'repo_name': 'felinblackcat/Trabajo1TAE2020', 'size': '7938'}