<jupyter_start><jupyter_text>random forest<jupyter_code>from sklearn.model_selection import train_test_split

def rf():
    sum_mse_train = 0
    sum_mse_test = 0
    for filename in filenames:
        filename_new = filename.replace('.csv','')
        line,stationID,stationDIR = filename_new.split('_')
        cur_dir = datasets_folder+'/'+filename
        data = pandas.read_csv(cur_dir,header=None)
        data.columns = ['month','date','hour','temp','pressure','humidity',
                       'wind_speed','wind_direction','clouds','weather_code',
                       'minute','delay']
        X = data[['hour','temp','pressure','humidity','wind_speed','clouds','weather_code','minute']]
        y = data.iloc[:,11]
        X_train,X_test, y_train, y_test = train_test_split(X,y,test_size=0.2)

        importances, mse, y_pred_test = model_fit(regr,X_train,X_test,y_train,y_test)
        sum_mse_train += mse
        sum_mse_test += mean_squared_error(y_test,y_pred_test)
        #print('new station')
        #print(importances)
    return sum_mse_train/len(filenames),sum_mse_test/len(filenames)

mse_train = 0
mse_test = 0
for i in range(10):
    train,test = rf()
    mse_train += train
    mse_test += test
print(mse_train/10)
print(mse_test/10)<jupyter_output>3.8862287167203027
4.043676663597375<jupyter_text>parameter selection<jupyter_code>from sklearn.model_selection import GridSearchCV
import json

dict_para = {}
for filename in filenames:
    filename_new = filename.replace('.csv','')
    line,stationID,stationDIR = filename_new.split('_')
    cur_dir = datasets_folder+'/'+filename
    data = pandas.read_csv(cur_dir,header=None)
    data.columns = ['month','date','hour','temp','pressure','humidity',
                   'wind_speed','wind_direction','clouds','weather_code',
                   'minute','delay']
    X = data[['hour','temp','pressure','humidity','wind_speed','clouds','minute']]
    y = data.iloc[:,11]
    X_train,X_test, y_train, y_test = train_test_split(X,y,test_size=0.2)

    tuned_parameters = [{'kernel':['rbf'],
                         'gamma':[1e-3,1e-4,1e-2,0.1,1],
                         'alpha':[0.1,1,10,100,1000]}]

    clf = GridSearchCV(KernelRidge(),tuned_parameters,cv=5)
    clf.fit(X_train,y_train)
    
    dict_para[filename_new] = clf.best_params_
model_param_dir = dir_path+'/models/model_param/krr_param.json'


with open(model_param_dir, 'w') as fp:
    json.dump(dict_para, fp)<jupyter_output><empty_output><jupyter_text>kernel ridge regression<jupyter_code>from sklearn.kernel_ridge import KernelRidge

def krr_fit(X_train,X_test,y_train,y_test,key):
    with open(model_param_dir, 'r') as fp:
        data = json.load(fp)
    
    kernel_param = data[key]['kernel']
    gamma_param = data[key]['gamma']
    alpha_param = data[key]['alpha']
    
    clf = KernelRidge(alpha=alpha_param,kernel = kernel_param,gamma = gamma_param)
    clf.fit(X_train, y_train)
    
    y_train_pred = clf.predict(X_train)
    y_test_pred = clf.predict(X_test)
    train_size = y_train_pred.shape
    test_size = y_test_pred.shape
    #fool_train = np.zeros(train_size)
    #fool_test = np.zeros(test_size)
    
    mse_train = mean_squared_error(y_train,y_train_pred)
    mse_test = mean_squared_error(y_test,y_test_pred)
    
    return mse_train, mse_test

def krr():
    sum_mse_train = 0
    sum_mse_test = 0
    for filename in filenames:
        filename_new = filename.replace('.csv','')
        line,stationID,stationDIR = filename_new.split('_')
        cur_dir = datasets_folder+'/'+filename
        data = pandas.read_csv(cur_dir,header=None)
        data.columns = ['month','date','hour','temp','pressure','humidity',
                       'wind_speed','wind_direction','clouds','weather_code',
                       'minute','delay']
        X = data[['hour','temp','pressure','humidity','wind_speed','clouds','minute']]
        y = data.iloc[:,11]
        X_train,X_test, y_train, y_test = train_test_split(X,y,test_size=0.2)

        mse_train, mse_test = krr_fit(X_train,X_test,y_train,y_test,filename_new)
        sum_mse_train += mse_train
        sum_mse_test += mse_test
        #print('new station')
        #print(importances)
    return sum_mse_train/len(filenames),sum_mse_test/len(filenames)

mse_train = 0
mse_test = 0
for i in range(5):
    train,test = krr()
    mse_train += train
    mse_test += test
print(mse_train/5)
print(mse_test/5)
LSTM (currently doing)
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
from keras.layers import Dropout

def rnn_regr(X_train,X_test,y_train,y_test)
    #Initialize the RNN
    regressor = Sequential()
    regressor.add(LSTM(units=50,return_sequences=True, input_shape=(X_train.shape[1],1)))
    regressor.add(Dropout(0.2))

    # add a second LSTM layer
    regressor.add(LSTM(units = 50, return_sequences = True))
    regressor.add(Dropout(0.2))

    # add a third LSTM layer
    regressor.add(LSTM(units = 50, return_sequences = True))
    regressor.add(Dropout(0.2))

    # add a forth LSTM layer
    regressor.add(LSTM(units = 50, return_sequences = False))
    regressor.add(Dropout(0.2))

    # add a output layer
    regressor.add(Dense(units = 1))

    # Compiling the RNN
    regressor.compile(optimizer = 'adam', loss = 'mean_squared_error')

    # Fitting the RNN to the Training set
    regressor.fit(X_train, y_train, epochs = 100, batch_size = 32)
    
    regressor.predict()
from sklearn.preprocessing import MinMaxScaler
sc = MinMaxScaler(feature_range = (0,1))

def dataset_reshape(scaled):
    X_train = []
    y_train = []
    
    length = scaled.shape[0]
    for i in range(60,length):
        X_train.append(scaled[i-60:i,0])
        y_train.append(scaled[i,0])
    X_train,y_train = np.array(X_train),np.array(y_train)
    X_train = np.reshape(X_train,(X_train.shape[0],X_train.shape[1],1))
    return X_train,y_train

for filename in filenames:
    filename_new = filename.replace('.csv','')
    line,stationID,stationDIR = filename_new.split('_')
    cur_dir = datasets_folder+'/'+filename
    data = pandas.read_csv(cur_dir,header=None)
    data.columns = ['month','date','hour','temp','pressure','humidity',
                   'wind_speed','wind_direction','clouds','weather_code',
                   'minute','delay']
    training_set = data.iloc[:,11:].values
    #training_set_reshape = training_set.reshape(-1,1)
    data_scaled = sc.fit_transform(training_set)
    train, test = train_test_split(data_scaled,test_size=0.1)
    X_train,y_train = dataset_reshape(train)
    X_test,y_test = dataset_reshape(test)<jupyter_output>/Users/michael/anaconda3/lib/python3.6/site-packages/sklearn/utils/validation.py:475: DataConversionWarning: Data with input dtype int64 was converted to float64 by MinMaxScaler.
  warnings.warn(msg, DataConversionWarning)
{'fraction_non_alphanumeric': 0.13465461964441855, 'fraction_numerical': 0.019819294666278055, 'mean_word_length': 3.829697396199859, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '348286', 'license': 'MIT', 'n_tokens_mistral': 2565, 'n_tokens_neox': 2442, 'n_words': 371, 'path': 'weather/.ipynb_checkpoints/feature_evaluation-checkpoint.ipynb', 'repo_name': 'rz2390/NYC-subway-delay-prediction', 'size': '6862'}