<jupyter_start><jupyter_text>資料預處理:能使用list（文字）和array（計算）就不要用pandas，太慢囉  觀察<jupyter_code>import numpy as np
import pandas as pd
import os

filepath = '/Users/mac/Desktop/Kaggle_datasets/Taxi_Duration/'
filename01 = 'train.csv'
filename02 = 'test.csv'
filename03 = 'sample_submission.csv'

df_train = pd.read_csv(os.path.join(filepath, filename01))
df_test = pd.read_csv(os.path.join(filepath, filename02))
df_ans = pd.read_csv(os.path.join(filepath, filename03))
df_train_copy = df_train
df_test_copy = df_test
df_train_copy.info()
df_train_copy
df_train_copy['delta_long'] = abs(df_train_copy['pickup_longitude']-df_train_copy['dropoff_longitude']) 
df_train_copy['delta_la'] = abs(df_train_copy['pickup_latitude']-df_train_copy['dropoff_latitude'])
df_train_copy['dist'] = np.sqrt(df_train_copy['delta_long']**2 + df_train_copy['delta_la']**2)

df_test_copy['delta_long'] = abs(df_test_copy['pickup_longitude']-df_test_copy['dropoff_longitude']) 
df_test_copy['delta_la'] = abs(df_test_copy['pickup_latitude']-df_test_copy['dropoff_latitude'])
df_test_copy['dist'] = np.sqrt(df_test_copy['delta_long']**2 + df_test_copy['delta_la']**2)
import matplotlib.pyplot as plt
plt.scatter(df_train_copy.dist, df_train_copy.trip_duration, alpha=0.1)
plt.show()
import seaborn as sns
import pandas as pd
import matplotlib.pyplot as plt

sns.jointplot(df_train_copy.dist, df_train_copy.trip_duration , data = df_train_copy)
plt.show()
df_train_copy[df_train_copy.dist>2]
df_train_copy[df_train_copy.trip_duration>3000]
df_train_copy['vendor_id'].value_counts()
df_train_copy['store_and_fwd_flag'].value_counts()
df_train_copy.columns
df_train_copy['trip_duration'].describe()
df_train_copy['dist'].describe()<jupyter_output><empty_output><jupyter_text>把上車時間轉換成數字，只取月份和小時  血淚教訓：List速度>>>pandas<jupyter_code>from datetime import datetime

t = datetime.strptime(df_train_copy['pickup_datetime'][0], '%Y-%m-%d %H:%M:%S')
df_train_copy['month'][0] = t.month
df_train_copy['hour'][0] = t.hour

print(df_train_copy['month'][0])
print(df_train_copy['hour'][0])
from datetime import datetime

pickup_datetime = df_train_copy['pickup_datetime'].tolist()
month = []
hour = []

for i in range(len(df_train_copy)):
    t = datetime.strptime(pickup_datetime[i],'%Y-%m-%d %H:%M:%S')
    month.append(t.month)
    hour.append(t.hour)
    
    if i % 100000 == 0:
        print('already: ', i)

df_train_copy['month'] = month
df_train_copy['hour'] = hour

df_train_copy
from datetime import datetime

pickup_datetime_test = df_test_copy['pickup_datetime'].tolist()
month = []
hour = []

for i in range(len(df_test_copy)):
    t = datetime.strptime(pickup_datetime_test[i],'%Y-%m-%d %H:%M:%S')
    month.append(t.month)
    hour.append(t.hour)
    
    if i % 100000 == 0:
        print('already: ', i)

df_test_copy['month'] = month
df_test_copy['hour'] = hour
df_test_copy<jupyter_output><empty_output><jupyter_text>整理所需參數<jupyter_code>df_train_para = df_train_copy[['vendor_id', 'passenger_count','store_and_fwd_flag',
                               'trip_duration','dist','month','hour']]

df_test_para = df_test_copy[['vendor_id', 'passenger_count','store_and_fwd_flag',
                             'dist','month','hour']]
df_train_para['month'].value_counts()
sns.boxplot(df_train_para['hour'], df_train_para['trip_duration'])
plt.show()
df_test_para['month'].value_counts()<jupyter_output><empty_output><jupyter_text>剔除outlier<jupyter_code>df_train_s = df_train_para[(df_train_para.trip_duration < 2500) &
                           (df_train_para.dist < 0.04) &
                           (df_train_para.dist > 0.01) ]
import matplotlib.pyplot as plt
plt.scatter(df_train_s.dist, df_train_s.trip_duration, alpha=0.01)
plt.show()
sns.boxplot(df_train_s['hour'], df_train_s['trip_duration'])
plt.show()
sns.boxplot(df_train_s['month'], df_train_s['trip_duration'])
plt.show()
sns.boxplot(df_train_s['passenger_count'], df_train_s['trip_duration'])
plt.show()
df_train_s['passenger_count'].value_counts()
df_train_s['trip_duration'].describe()
df_train_s['dist'].describe()
df_test_para
df_test_para['vendor_id'].value_counts()
df_test_para['store_and_fwd_flag'].value_counts()
df_test_para['dist'].describe()
df_ans.head()
df_ans[df_ans.id == 0]<jupyter_output><empty_output><jupyter_text>製作feature和labels：vendor,flag,month,hour做onehot_encoding<jupyter_code>df_train_feature = df_train_s[['vendor_id','store_and_fwd_flag','dist','month','hour']]
df_train_label = df_train_s['trip_duration']

df_test_feature = df_test_para[['vendor_id','store_and_fwd_flag','dist','month','hour']]
df_train_feature = pd.get_dummies(data=df_train_feature, 
                                  columns=['vendor_id','store_and_fwd_flag',
                                           'month','hour'])

df_test_feature = pd.get_dummies(data=df_test_feature, 
                                  columns=['vendor_id','store_and_fwd_flag',
                                           'month','hour'])
df_train_feature.head()
df_train_feature.columns
df_train_feature['dist'].describe()
df_test_feature.head()
df_test_feature.columns
df_test_feature['dist'].describe()
train_feature = df_train_feature.values
test_feature = df_test_feature.values
train_label = df_train_label.values
from sklearn import preprocessing
min_max_scaler = preprocessing.MinMaxScaler()
train_feature_trans = min_max_scaler.fit_transform(train_feature.reshape(-1,35))
test_feature_trans = min_max_scaler.fit_transform(test_feature.reshape(-1,35))
train_feature_trans[0]
train_feature.shape
test_feature_trans[0]
train_label = train_label.reshape(-1,1)
train_label<jupyter_output><empty_output><jupyter_text>跑模型囉！！<jupyter_code>import matplotlib.pyplot as plt
def show_train_history(train_history,train,validation):
    plt.plot(train_history.history[train])
    plt.plot(train_history.history[validation])
    plt.title('Train History')
    plt.ylabel(train)
    plt.xlabel('Epoch')
    plt.legend(['train', 'validation'], loc='upper left')
    plt.show()


######################### 建立模型
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout
from keras.models import load_model

model = Sequential()

model.add(Dense(units=100, 
                input_dim=35,
                kernel_initializer='uniform'))
model.add(Dropout(0.5))

model.add(Dense(units=100,
                kernel_initializer='uniform'))
model.add(Dropout(0.5))

model.add(Dense(units=50,
                kernel_initializer='uniform'))
model.add(Dropout(0.5))

model.add(Dense(units=1))

print(model.summary())

#可以反覆訓練fine tune
weights_path = 'Savemodels/TaxiDuration(Kaggles)_MLP.h5'
model.load_weights(weights_path)

model.compile(loss='mean_squared_logarithmic_error',   
              optimizer='adam', metrics=['accuracy'])

train_history = model.fit(train_feature, train_label,
                          validation_split=0.2, epochs=20, batch_size=50000, verbose=2) 


######################### 訓練過程視覺化
show_train_history(train_history,'acc','val_acc')
show_train_history(train_history,'loss','val_loss')

#儲存訓練結果
model.save_weights("Savemodels/TaxiDuration(Kaggles)_MLP.h5")
print('model saved to disk')
prediction = model.predict(test_feature)
import matplotlib.pyplot as plt
plt.hist(prediction, range=(0,1500), bins = 20)
plt.show()
prediction[0]
prediction<0
df_ans['trip_duration'] = prediction
df_ans
df_ans['trip_duration'].describe()
df_ans[df_ans['trip_duration']>900]
df_ans[df_ans.trip_duration<0]
ans = df_ans['trip_duration'].tolist()

for i in range(len(ans)):
    if ans[i] < 0:
        ans[i] = 1
        
df_ans['trip_duration'] = ans
df_ans[df_ans['id']==0]
df_ans.to_csv('TaxiDuration_ans.csv',mode = 'w', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1488782259110362, 'fraction_numerical': 0.014395020101154196, 'mean_word_length': 6.010909090909091, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '594054', 'license': 'MIT', 'n_tokens_mistral': 3009, 'n_tokens_neox': 2905, 'n_words': 343, 'path': 'Keras_TaxiDuration(Kaggles)_MLP.ipynb', 'repo_name': 'Pytoddler/Kaggle-competition', 'size': '7711'}