<jupyter_start><jupyter_text>* windows개념 이용해서 univariate time series forecasting의 방법으로 해결해보고자 함<jupyter_code># Runned_by_Colab

# 1. 이번 Trial 내용
# ReduceLROnPlateau() <- loss값에 따라 학습률 자동으로 조정해주는 매서드
# 각 sample_id마다 test_x데이터로 학습할 때, 학습 이후 줄어든 최종 loss값과 learning rate 그리고 sample_id 출력하도록 설정
# input: 60, output: 1의 many-to-one 모델<jupyter_output><empty_output><jupyter_text>---<jupyter_code>#Import Library
import tensorflow as tf
physical_devices = tf.config.list_physical_devices('GPU')
tf.config.experimental.set_memory_growth(physical_devices[0], True)
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, LSTM, TimeDistributed, Dropout, Bidirectional#,Reshape, Flatten
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from tqdm import tqdm
import warnings
warnings.filterwarnings("ignore")
# Read Data
data_path = '../Big_Data/[DACON]Bit_Trader'
train_x_df = pd.read_csv(data_path  + "/train_x_df.csv")
train_y_df = pd.read_csv(data_path  + "/train_y_df.csv")
test_x_df = pd.read_csv(data_path  + "/test_x_df.csv")
# 입력 받은 2차원 데이터 프레임을 3차원 numpy array로 변경하는 함수
def df2d_to_array3d(df_2d):
    feature_size = df_2d.iloc[:,2:].shape[1]
    time_size = len(df_2d.time.value_counts())
    sample_size = len(df_2d.sample_id.value_counts())
    array_3d = df_2d.iloc[:,2:].values.reshape([sample_size, time_size, feature_size])
    return array_3d

# 2차원 DF LSTM으로 학습하기 위해 3차원으로 변환시키기
train_x_array = df2d_to_array3d(train_x_df)   #(1380, 1380, 10)
train_y_array = df2d_to_array3d(train_y_df)   #(1380, 120, 10)
test_x_array = df2d_to_array3d(test_x_df)     #(529, 1380, 10)
# 모델 구성(op1: many-to-one model, op2: many-to-many model(output 모양 바꾸기, return_sequences = True, Window이동 단위 1에서 output 크기로 변경))
# LSTM에 return_sequence=True가 주어지고 Bidirectional이면서 TimeDistributed() 함수가 사용되면 양방향 many-to-many 유형이 된다.https://m.blog.naver.com/chunjein/221589656211
# TimeDistributed() 함수 사용: 모델 layer에 model.add(TimeDistributed(Dense(1, activation='sigmoid')))추가

def build_model():
    seq_len = 60
    model = Sequential()
    # 첫 번째 layer에서 LSTM()매서드 안의 stateful = True는 오히려 다운 퍼포먼스 될 수 있음
    model.add(LSTM(50, activation='tanh', return_sequences= True, input_shape = [seq_len, 1], dropout = 0.2)) 
#     model.add(Bidirectional(LSTM(50, activation='tanh', dropout=0.50, recurrent_dropout=0.25)))
    model.add(LSTM(50, activation= 'tanh', dropout = 0.2))
    model.add(Dense(1))  #60개의 open값 입력되어 121번째 open값 '하나' 예측   
    
    model.compile(optimizer = 'adam', loss = 'mse', metrics = ['mse'])
    
    return model
build_model().summary()
# LearningRateScheduler(scheduler) 구현

# from tensorflow.keras.callbacks import LearningRateScheduler
# # LearningRateScheduler 함수 생성 (epoch이 5이하면 그대로 lr 유지, 아닐시 lr 조정)
# def scheduler(epoch, learning_rate):
#     if epoch < 5:
#         return learning_rate
#     else:
#         return learning_rate * tf.math.exp(-0.1)
# lr = LearningRateScheduler(scheduler)
# test_x_array에 대한 Auto_Regressive한 Prediction 및 valid_pred_array에 예측 결과 기록

# 1) test_pred_array{예측값 모아두는 3차원 배열(120*1 2차원 배열 529개)} 만들기
test_pred_array = np.zeros([len(test_x_array), 120, 1])

# 2) early_stoppage & reduceLR 정의: https://www.dacon.io/competitions/official/235709/codeshare/2453?page=1&dtype=recent 참고)
early_stop = tf.keras.callbacks.EarlyStopping(monitor='loss', patience= 10, mode = 'auto')
reduceLR = tf.keras.callbacks.ReduceLROnPlateau(monitor='loss', factor=0.1, patience= 5, min_lr=0)

# 3) test_x_array로 시계열 Windows 만들기 -> 데이터셋 구성 -> 모델 학습 ||| 예측 -> test_pred_array에 기록 -> window_3d의 첫번째 값 삭제 -> test_pred_array와 window_3d 병합 -> model.predict()에 넣어 예측 -> ***
ep = 30
bs = 120
# idx에 해당하는 샘플 529개 학습: for loop 529번 돌아감
for idx in tqdm(range(test_x_array.shape[0])):  # 529번
    seq_len = 60
    sequence_length = seq_len + 1

    windows = []
    for index in range(1380 - sequence_length):
        windows.append(test_x_array[idx, :, 1][index: index + sequence_length])

    # x_test, y_test 데이터셋 구성
    windows = np.array(windows)  # 1329 * 121의 2차원 배열
    x_test = windows[:, :-1]
    x_test = np.reshape(x_test, (x_test.shape[0], x_test.shape[1], 1))
    y_test = windows[:, -1]
    
    model = build_model()
    history = model.fit(x_test, y_test, epochs= ep, batch_size= bs, verbose=0, shuffle = True, callbacks=[early_stop, reduceLR])
# shuffle= True 효과 있을까? 미약하게나마 효과 있음. https://machinelearningmastery.com/stateful-stateless-lstm-time-series-forecasting-python/
    print('sample_id : ', idx, '번')
    print('loss : ', history.history['loss'][-1])
#     print('mse : ', history.history['mse'][-1])  #<- model.compile()의 metrics인자값으로 설정된 값을 추가로 출력해 주고 싶다면!
    print('lr : ', round(model.optimizer.lr.numpy(), 5))  #<-조정된 학습률 출력

    # test_x_array Windows 중 마지막 윈도우 추출해서 3차원 변환시켜 LSTM모델에 넣고 Predict
    window = windows[-1, :-1]  # windows.shape (1259, 61), window.shape (60, )
    window_3d = np.reshape(window, (1, window.shape[0], 1))  # (1, 60, 1)
    for m in range(120):
        # model.predict()에 window_3d 넣어 예측
        pred = model.predict(window_3d)

        # 120분 중 처음 1분 예측값 test_pred_array에 기록
        test_pred_array[idx, m, :] = pred

        # window_3d의 첫번째 분 값을 삭제한 window_3d_2nd 구성
        window_3d_2nd = window_3d[0, 1:, :]  # 119개 -> 118개 -> ***

        # pred_target(prediction할 때마다 나오는 각각의 예측값들) 1차원 -> 2차원으로 구성
        pred_target = test_pred_array[idx, m, :]
        pred_target = np.reshape(pred_target, (pred_target.shape[0], 1))

        # test_pred_array와 window_3d_2nd 병합하여 모델에 입력할 새로운 window_3d 재구성
        window_3d = np.concatenate((window_3d_2nd, pred_target), axis=0)
        window_3d = window_3d.T
        window_3d = np.reshape(window_3d, (window_3d.shape[0], window_3d.shape[1], 1))   
# 손실값 시각화
plt.plot(history.history['loss'], 'b-', label='loss')
plt.xlabel('Epoch')
plt.legend()
plt.show()
# test_x_array 각 샘플의 2시간 예측값이 기록되어 있는 test_pred_array shape 확인
print(test_pred_array.shape)
# 모델 저장 및 로드
model.save('./my_model.h5')
model = tf.keras.models.load_model('./my_model.h5')
# 매수 시점, 매수 비율 표 만들기
# 1) train_pred_array 3차원에서 2차원으로 바꾸기
pred_array_2d = np.zeros([test_pred_array.shape[0], 120])

for idx in tqdm(range(test_pred_array.shape[0])):
    pred_array_2d[idx, :] = test_pred_array[idx, :, 0]

# 2) 예측값을 재해석하여 submission 표를 작성하는 함수 정의
def array_to_submission(pred_array):
    submission = pd.DataFrame(np.zeros([pred_array.shape[0], 2], np.int64),
                              columns=['buy_quantity', 'sell_time'])
    submission = submission.reset_index()
    sell_price = []
    for idx, sell_time in enumerate(np.argmax(pred_array, axis=1)):
        sell_price.append(pred_array[idx, sell_time])
    sell_price = np.array(sell_price)
    submission.loc[:, 'buy_quantity'] = ((1*1*(sell_price/1)*0.9995*0.9995) > 1.08)*1 #-DACON-Bit_Trader폴더 주가 손실계산.png 참고
    submission['sell_time'] = np.argmax(pred_array, axis=1)
    submission.columns = ['sample_id', 'buy_quantity', 'sell_time']
    return submission, sell_price

final_submission, forcasted_max = array_to_submission(pred_array_2d)
# 각 샘플의 예측치 중 최고값 모아 보기
forcasted_max
# 전체 300가지 sample에 대해 _가지 case에서 (수수료 감안해서) 매수 시점(1380분)보다 108% 이상 상승한다고 추론함.
final_submission.buy_quantity.value_counts()
# (수수료 감안하지 않고) 매수 시점보다
forcasted_max[forcasted_max >= 1.08]
# final_submission csv파일로 저장
final_submission.to_csv('./submission.csv', index = False)<jupyter_output><empty_output><jupyter_text>---<jupyter_code>forcasted_max = np.array([1.0065794 , 1.01441026, 0.93679267, 0.95263463, 0.9962129 ,
       1.04470217, 0.96156454, 0.96337712, 0.99317759, 0.98923784,
       1.0368427 , 0.96484196, 1.0166893 , 0.98878568, 0.9972856 ,
       1.02150273, 1.02592289, 0.98527056, 1.02538037, 1.02668178,
       1.01431894, 0.97106212, 1.01990569, 1.09836495, 1.02756822,
       1.10469329, 1.08048487, 0.96482199, 0.97250688, 0.98419917,
       0.9925257 , 1.02387679, 1.04667366, 0.96720475, 1.00631928,
       1.03889847, 0.97621673, 0.96684706, 1.0171299 , 1.0311259 ,
       0.92322177, 0.9835428 , 1.00146222, 1.0395757 , 0.9189626 ,
       1.01740444, 0.99568838, 1.00893879, 0.98794138, 1.03709316,
       1.03929853, 0.96503383, 1.01527643, 1.00529766, 0.98389959,
       0.96083832, 0.97983682, 0.99698496, 1.02432573, 0.99810266,
       1.11322498, 0.98820484, 1.03982139, 0.91585803, 1.00948894,
       0.96425682, 0.96712118, 0.99290192, 1.11809063, 0.97789603,
       1.00969374, 1.03134739, 1.05668056, 0.92782664, 0.9845044 ,
       1.00621986, 1.0416224 , 1.01342475, 0.96747202, 1.11189604,
       0.99699718, 1.06782651, 1.02171659, 0.98289645, 0.99201769,
       0.93830907, 0.94877851, 0.93801427, 0.91771084, 1.03319883,
       0.94863939, 1.01777411, 0.98115009, 0.97849214, 0.98872936,
       0.99182302, 1.02347171, 1.27607596, 1.0195961 , 0.98061615,
       1.13997781, 0.9999162 , 1.10762215, 1.02317441, 0.99676895,
       1.02686906, 0.98811936, 1.03384686, 0.98263282, 1.03438771,
       1.00222373, 0.97913915, 1.02264297, 0.97258198, 1.03133857,
       1.03502476, 0.94605422, 0.97657591, 1.05702603, 1.0154866 ,
       1.17250073, 0.97349459, 0.97367352, 1.07784235, 1.07601416,
       1.02025807, 1.03213131, 0.99918586, 0.97095579, 0.94548917,
       0.97960836, 1.00274539, 1.00798011, 0.93696493, 1.00060105,
       0.96673054, 0.96844155, 0.96408051, 1.00158787, 0.95024079,
       1.02234292, 1.04485726, 1.00018692, 0.93890339, 1.14777517,
       0.93857306, 0.99466604, 0.97476375, 1.00672567, 1.0568161 ,
       1.1660763 , 0.88703239, 0.99038076, 0.96688014, 0.95366609,
       0.90003937, 0.9770909 , 1.0229634 , 1.04560184, 0.96615678,
       0.8935414 , 1.00266254, 0.97203422, 1.00107861, 0.9718684 ,
       0.97999078, 0.96193993, 0.97848368, 1.03321898, 0.99827588,
       0.97643536, 0.93810105, 0.95938355, 0.94542319, 0.99423003,
       0.95409453, 1.02544117, 0.95914549, 1.00326693, 0.97777754,
       0.9707188 , 1.02591383, 1.03447115, 0.98033702, 1.01765978,
       0.95761555, 1.01251733, 0.98720646, 1.00226784, 1.04576361,
       0.98661673, 1.03522348, 1.02343714, 0.98901081, 0.96577662,
       0.99879533, 0.95419908, 1.02630496, 0.98549771, 1.00768209,
       1.00372422, 1.02120733, 0.99340057, 1.03806603, 0.96339053,
       0.9210785 , 0.98824489, 0.9859097 , 1.03655267, 1.03060508,
       0.99525303, 1.03708541, 0.97912896, 0.97516572, 0.96538931,
       0.97141331, 0.9920525 , 1.08049989, 0.97312886, 0.95255375,
       0.96016711, 0.94763523, 1.01296294, 1.06056917, 1.03816271,
       1.01066661, 0.95114905, 1.0177927 , 1.06653225, 0.96923971,
       1.02120328, 0.9365378 , 1.02398062, 0.98482251, 0.96432841,
       0.99984151, 0.97617221, 1.03808677, 0.99515945, 1.04617739,
       1.03970754, 1.01118255, 0.97288215, 0.9649905 , 1.04797077,
       1.04025972, 0.97705477, 0.9107464 , 0.92247373, 0.93615246,
       1.010288  , 0.96398365, 0.93465853, 0.91844207, 0.99726766,
       0.95257443, 1.0502615 , 0.94940001, 1.10219347, 0.92839134,
       1.02199888, 1.06174123, 1.04553688, 0.99459356, 1.00422931,
       0.98570293, 1.06866157, 1.04643929, 1.03694642, 0.96084201,
       0.98886162, 1.00142455, 0.96943659, 0.97481221, 1.01615286,
       0.99443656, 1.02085412, 1.03036809, 0.89826638, 0.91090339,
       1.02495694, 1.03306961, 0.97037494, 1.02033865, 0.9547053 ,
       1.00408292, 0.92122602, 1.00014377, 0.9138217 , 1.17342126,
       0.98702341, 0.97080821, 1.01818001, 1.02643812, 1.02400208,
       0.98407573, 0.98586106, 1.03314388, 0.96764559, 1.04155123,
       1.0510025 , 1.04115701, 1.00050068, 1.00013053, 0.99006903,
       0.97704297, 1.02716279, 1.08919168, 0.96974373, 0.98965448,
       0.99512869, 1.01098609, 1.05249393, 1.03907919, 0.97517997,
       0.99773848, 1.00783241, 0.99467748, 0.97996062, 0.96273106,
       0.92750388, 1.02492476, 0.92230302, 1.01267707, 1.01905107,
       1.05797386, 1.01009405, 1.01231599, 0.9776147 , 1.02381933,
       0.96404946, 0.94987875, 1.04162633, 0.93252927, 1.01455617,
       1.01413465, 1.01518691, 0.97460037, 0.96769428, 1.05115032,
       1.00932288, 0.98593336, 1.02005112, 0.995049  , 0.94258904,
       1.01796222, 0.99506086, 1.04774129, 0.99144489, 1.0356338 ,
       1.01098585, 1.07483006, 0.98593611, 1.03039372, 1.02956915,
       0.967215  , 0.99972886, 0.97976458, 0.97874039, 1.00588489,
       1.02787364, 1.00336862, 0.91790944, 1.05879676, 0.97168559,
       0.99148655, 1.03463459, 0.96508694, 0.99922377, 1.04803562,
       1.01558387, 1.02605355, 1.00572085, 0.99389392, 0.98456013,
       1.03268039, 1.00479627, 1.0208019 , 1.02020717, 1.00788879,
       1.05442953, 1.04218519, 0.99493945, 1.02787387, 0.988424  ,
       1.05971086, 1.02237535, 1.00735855, 0.96601766, 0.94491804,
       1.01114297, 1.03632009, 0.94412708, 0.99503708, 0.97809935,
       0.98321795, 0.97160083, 1.04218566, 1.04665279, 0.95321572,
       0.94863552, 1.02721131, 1.00807416, 0.96781802, 0.96433783,
       0.9284004 , 0.98689222, 1.04078221, 1.15119946, 1.02979684,
       0.97553766, 1.09933197, 1.07495248, 1.03281367, 0.96915239,
       1.04592109, 1.05572629, 1.0861485 , 0.96019322, 1.02414668,
       0.95657051, 0.95507491, 0.94674671, 0.97850597, 1.07136321,
       0.97467685, 0.97042698, 1.06747854, 1.05773056, 0.9893176 ,
       1.00414014, 1.02339673, 1.03197241, 1.00046492, 1.0214994 ,
       0.98325944, 0.98257494, 0.99025577, 0.97856146, 1.00944006,
       0.93256855, 0.98230743, 0.95875198, 1.02652252, 1.06346452,
       0.99673867, 1.06114435, 1.01973116, 0.92666554, 1.0254364 ,
       0.99466819, 1.07786548, 0.99847555, 0.99067497, 1.06686962,
       0.97913909, 0.9859103 , 1.05641925, 0.94291908, 0.99300295,
       1.0311029 , 1.00482225, 0.96212506, 1.05377471, 0.99549294,
       1.0022341 , 1.07437372, 1.00378513, 1.03888071, 0.9665693 ,
       1.04088521, 0.93599629, 1.04600418, 0.97741258, 1.04731905,
       0.95125884, 0.93689406, 0.98672026, 0.98503309, 1.00422812,
       0.94472957, 0.97476602, 1.02272463, 0.94372642, 1.00191963,
       0.99464881, 0.9948234 , 1.01057267, 0.9933421 , 0.98892772,
       0.96496403, 1.01080871, 0.92618644, 0.94617021, 1.01859796,
       1.04663002, 1.00299931, 1.03730428, 0.98753816, 0.98874718,
       0.99014807, 0.96099061, 0.971973  , 0.99697995, 1.00956547,
       1.02836859, 1.0232929 , 1.01373661, 0.94604689, 0.98887771,
       0.98924828, 0.99897289, 0.94501275, 0.9566682 , 1.01734698,
       1.01776481, 0.97679836, 1.04253197, 1.00672615, 1.04028904,
       0.96025473, 1.05315185, 0.97637963, 1.03204584, 0.94754422,
       1.0106324 , 1.00989068, 0.99018216, 0.99898666])
len(forcasted_max)
l = []
for i in forcasted_max[forcasted_max>=1.13]:
    l.append(i)
l
for j in l:
    print(np.where(forcasted_max == j))
# 모델 평가: test_x데이터로 예측하는 방식을 입력값(train_x)에 대한 예측값과 실제값(train_y_array) 비교를 통해 평가

train_pred_array = np.zeros([1, 120, 1])

sample = 70  #평가용 임의의 샘플 id

early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience= 10, mode = 'auto')
reduceLR = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience= 3, min_lr = 0)

# train_x_array데이터로 시계열 Windows 만들기
# idx에 해당하는 샘플 하나만 학습하므로 for loop는 한 번만 돌아감
ep = 30
bs = 120
for idx in range(sample, sample+1):
    seq_len = 60  # window_size와 같은 개념
    sequence_length = seq_len + 1

    windows = []
    for index in range(1380 - sequence_length):
        windows.append(train_x_array[idx, :, 1][index: index + sequence_length])

    # x_train, y_train 데이터 구성
    windows = np.array(windows) #1329 * 121의 2차원 배열
    x_train = windows[:, :-1]
    x_train = np.reshape(x_train, (x_train.shape[0], x_train.shape[1], 1))
    y_train = windows[:, -1]

    
    # Fit(early_stoppage & reduceLR 적용)
    model = build_model()
    history = model.fit(x_train, y_train, validation_split=0.1, epochs = ep, batch_size = bs, verbose = 2, callbacks = [early_stop, reduceLR])
    print('sample_id : ', idx, '번')
    print('loss : ', history.history['loss'][-1])
#     print('mse : ', history.history['mse'][-1])  #<- model.compile()의 metrics인자값으로 설정된 값을 추가로 출력해 주고 싶다면!
    print('lr : ', round(model.optimizer.lr.numpy(), 5))  #<-조정된 학습률 출력
    
    # train_x_array Windows 중 마지막 윈도우 추출해서 3차원 변환시켜 LSTM모델에 넣고 Predict
    window = windows[-1, :-1]
    window_3d = np.reshape(window, (1, window.shape[0], 1))
    for m in range(120):
        # model.predict()에 window_3d 넣어 예측
        pred = model.predict(window_3d)

        # 120분 중 처음 1분 예측값 train_pred_array에 기록
        train_pred_array[:, m, :] = pred

        # window_3d의 첫번째 분 값은 삭제한 window_3d_2nd 구성
        window_3d_2nd = window_3d[0, 1:, :]  # 119개

        # pred_target(prediction할 때마다 나오는 각각의 예측값들) 1차원 -> 2차원으로 구성
        pred_target = train_pred_array[:, m, :]
        pred_target = np.reshape(pred_target, (pred_target.shape[0], 1))

        # train_pred_array와 window_3d_2nd 병합하여 모델에 입력할 새로운 window_3d 재구성
        window_3d = np.concatenate((window_3d_2nd, pred_target), axis=0)
        window_3d = window_3d.T
        window_3d = np.reshape(window_3d, (window_3d.shape[0], window_3d.shape[1], 1))
# train 샘플 훈련 성과 시각화해보기
# 1) 입력 series와 출력 series를 연속적으로 연결하여 시각적으로 보여주는 코드 정의
def plot_series(x_series, y_series):
    plt.plot(x_series, label = 'input_series')
    plt.plot(np.arange(len(x_series), len(x_series)+len(y_series)),
             y_series, label = 'output_series')
    plt.axhline(1, c = 'red')
    plt.legend()

# 2) train data 중 sample_id idx에 해당하는 x_series로 모델을 학습한 후 y_series를 추론
x_series = train_x_array[sample,:,1]
y_series = train_y_array[sample,:,1]
plot_series(x_series, y_series)
plt.plot(np.arange(1380, 1380+120), train_pred_array[0,:,0], label = 'prediction')
plt.legend()
plt.show()
# 손실값 시각화
plt.plot(history.history['loss'], 'b-', label='loss')
plt.plot(history.history['val_loss'], 'r--', label='val_loss')
plt.xlabel('Epoch')
plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>---<jupyter_code># train_x_df의 자료들을 학습에 활용하게 될 경우:
# train_x_df를 sample_id을 기준으로 추출하는 방법
train_x_df = train_x_df[train_x_df.sample_id < 300]
train_y_df = train_y_df[train_y_df.sample_id < 300]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1386731750610162, 'fraction_numerical': 0.28339250055469273, 'mean_word_length': 3.521946325558064, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 31, 'https://': 3, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '461446', 'license': 'MIT', 'n_tokens_mistral': 11720, 'n_tokens_neox': 9254, 'n_words': 1497, 'path': 'Trials/Bit_Trader_Trial2(13705).ipynb', 'repo_name': 'gi3265/-DACON-Bit_Trader', 'size': '18028'}