<jupyter_start><jupyter_text><jupyter_code>from google.colab import drive
drive.mount('/content/drive')
import numpy as np
import pandas as pd
# load data

dataX=pd.read_csv("/content/drive/My Drive/traindata/trainX.csv")
dataA=pd.read_csv("/content/drive/My Drive/traindata/trainA.csv")
dataY=pd.read_csv("/content/drive/My Drive/traindata/trainY.csv")

# remove index

dataX=dataX.values[:,1:]
dataY=dataY.values[:,1:-1]
dataA=dataA.values[:,1:]
np.where(dataA==1)[0]
dataX.shape
# split cure or not cure

data_CX=dataX[np.where(dataA==1)[0]]
dataNCX=dataX[np.where(dataA!=1)[0]]
data_CY=dataY[np.where(dataA==1)]
dataNCY=dataY[np.where(dataA!=1)]
data_CX
data_CY
from sklearn.model_selection import train_test_split

train_data_C,test_data_C,train_targets_C,test_targets_C=train_test_split(data_CX,data_CY)
train_dataNC,test_dataNC,train_targetsNC,test_targetsNC=train_test_split(data_CX,data_CY)

mean_C = train_data_C.mean(axis=0)
train_data_C -= mean_C
std_C = train_data_C.std(axis=0)
train_data_C /= std_C

test_data_C -= mean_C
test_data_C /= std_C


meanNC = train_dataNC.mean(axis=0)
train_dataNC -= meanNC
stdNC = train_dataNC.std(axis=0)
train_dataNC /= stdNC

test_dataNC -= meanNC
test_dataNC /= stdNC
from keras import models
from keras import layers

def build_model_C():
    # 동일한 모델을 여러 번 생성할 것이므로 함수를 만들어 사용합니다
    model = models.Sequential()
    model.add(layers.Dense(64, activation='relu',
                           input_shape=(train_data_C.shape[1],)))
    model.add(layers.Dense(64, activation='relu'))
    model.add(layers.Dense(1))
    model.compile(optimizer='rmsprop', loss='mse', metrics=['mean_absolute_error'])
    return model


def build_modelNC():
    # 동일한 모델을 여러 번 생성할 것이므로 함수를 만들어 사용합니다
    model = models.Sequential()
    model.add(layers.Dense(64, activation='relu',
                           input_shape=(train_dataNC.shape[1],)))
    model.add(layers.Dense(64, activation='relu'))
    model.add(layers.Dense(1))
    model.compile(optimizer='rmsprop', loss='mse', metrics=['mean_absolute_error'])
    return model

def build_model_C_100():
  # 2. 모델의 구조를 BatchNormalization layer를 사용하여 만든다.
  X = tf.keras.layers.Input(shape=[17])
  
  H = tf.keras.layers.Dense(8)(X)
  H = tf.keras.layers.BatchNormalization()(H)
  H = tf.keras.layers.Activation('swish')(H)
  
  H = tf.keras.layers.Dense(8)(H)
  H = tf.keras.layers.BatchNormalization()(H)
  H = tf.keras.layers.Activation('swish')(H)
  
  H = tf.keras.layers.Dense(8)(H)
  H = tf.keras.layers.BatchNormalization()(H)
  H = tf.keras.layers.Activation('swish')(H)
  
  Y = tf.keras.layers.Dense(1)(H)
  model = tf.keras.models.Model(X, Y)
  model.compile(loss='mse', metrics=[tf.keras.metrics.Accuracy()])
  return model

def build_model_C_100_16():
  # 2. 모델의 구조를 BatchNormalization layer를 사용하여 만든다.
  X = tf.keras.layers.Input(shape=[17])
  
  H = tf.keras.layers.Dense(16)(X)
  H = tf.keras.layers.BatchNormalization()(H)
  H = tf.keras.layers.Activation('swish')(H)
  
  H = tf.keras.layers.Dense(16)(H)
  H = tf.keras.layers.BatchNormalization()(H)
  H = tf.keras.layers.Activation('swish')(H)
  
  H = tf.keras.layers.Dense(16)(H)
  H = tf.keras.layers.BatchNormalization()(H)
  H = tf.keras.layers.Activation('swish')(H)
  
  Y = tf.keras.layers.Dense(1)(H)
  model = tf.keras.models.Model(X, Y)
  model.compile(loss='mse', metrics=[tf.keras.metrics.Accuracy()])
  return model

def build_model_NC_100():
  # 2. 모델의 구조를 BatchNormalization layer를 사용하여 만든다.
  X = tf.keras.layers.Input(shape=[17])
  
  H = tf.keras.layers.Dense(8)(X)
  H = tf.keras.layers.BatchNormalization()(H)
  H = tf.keras.layers.Activation('swish')(H)
  
  H = tf.keras.layers.Dense(8)(H)
  H = tf.keras.layers.BatchNormalization()(H)
  H = tf.keras.layers.Activation('swish')(H)
  
  H = tf.keras.layers.Dense(8)(H)
  H = tf.keras.layers.BatchNormalization()(H)
  H = tf.keras.layers.Activation('swish')(H)
  
  Y = tf.keras.layers.Dense(1)(H)
  model = tf.keras.models.Model(X, Y)
  model.compile(loss='mse', metrics=[tf.keras.metrics.Accuracy()])
  return model

def build_model_NC_100_16():
  # 2. 모델의 구조를 BatchNormalization layer를 사용하여 만든다.
  X = tf.keras.layers.Input(shape=[17])
  
  H = tf.keras.layers.Dense(16)(X)
  H = tf.keras.layers.BatchNormalization()(H)
  H = tf.keras.layers.Activation('swish')(H)
  
  H = tf.keras.layers.Dense(16)(H)
  H = tf.keras.layers.BatchNormalization()(H)
  H = tf.keras.layers.Activation('swish')(H)
  
  H = tf.keras.layers.Dense(16)(H)
  H = tf.keras.layers.BatchNormalization()(H)
  H = tf.keras.layers.Activation('swish')(H)
  
  Y = tf.keras.layers.Dense(1)(H)
  model = tf.keras.models.Model(X, Y)
  model.compile(loss='mse', metrics=[tf.keras.metrics.Accuracy()])
  return model

# cure

k = 4
num_val_samples = len(train_data_C) // k
num_epochs = 100
all_scores_C = []
for i in range(k):
    print('처리중인 폴드 #', i)
    # 검증 데이터 준비: k번째 분할
    val_data = train_data_C[i * num_val_samples: (i + 1) * num_val_samples]
    val_targets = train_targets_C[i * num_val_samples: (i + 1) * num_val_samples]

    # 훈련 데이터 준비: 다른 분할 전체
    partial_train_data = np.concatenate(
        [train_data_C[:i * num_val_samples],
         train_data_C[(i + 1) * num_val_samples:]],
        axis=0)
    partial_train_targets = np.concatenate(
        [train_targets_C[:i * num_val_samples],
         train_targets_C[(i + 1) * num_val_samples:]],
        axis=0)

    # 케라스 모델 구성(컴파일 포함)
    model_C = build_model_C()
    # 모델 훈련(verbose=0 이므로 훈련 과정이 출력되지 않습니다)
    model_C.fit(partial_train_data, partial_train_targets,
              epochs=num_epochs, batch_size=1, verbose=0)
    # 검증 세트로 모델 평가
    val_mse, val_mae = model_C.evaluate(val_data, val_targets, verbose=0)
    all_scores_C.append(val_mae)
# not cure

k = 4
num_val_samples = len(train_dataNC) // k
num_epochs = 100
all_scoresNC = []
for i in range(k):
    print('처리중인 폴드 #', i)
    # 검증 데이터 준비: k번째 분할
    val_data = train_dataNC[i * num_val_samples: (i + 1) * num_val_samples]
    val_targets = train_targetsNC[i * num_val_samples: (i + 1) * num_val_samples]

    # 훈련 데이터 준비: 다른 분할 전체
    partial_train_data = np.concatenate(
        [train_dataNC[:i * num_val_samples],
         train_dataNC[(i + 1) * num_val_samples:]],
        axis=0)
    partial_train_targets = np.concatenate(
        [train_targetsNC[:i * num_val_samples],
         train_targetsNC[(i + 1) * num_val_samples:]],
        axis=0)

    # 케라스 모델 구성(컴파일 포함)
    modelNC = build_modelNC()
    # 모델 훈련(verbose=0 이므로 훈련 과정이 출력되지 않습니다)
    modelNC.fit(partial_train_data, partial_train_targets,
              epochs=num_epochs, batch_size=1, verbose=0)
    # 검증 세트로 모델 평가
    val_mse, val_mae = modelNC.evaluate(val_data, val_targets, verbose=0)
    all_scoresNC.append(val_mae)
all_scores_C,np.mean(all_scores_C)

all_scoresNC,np.mean(all_scoresNC)

from keras import backend as K

# 메모리 해제
K.clear_session()
# cure

num_epochs = 1000
all_mae_histories_C = []
for i in range(k):
    print('처리중인 폴드 #', i)
    # 검증 데이터 준비: k번째 분할
    val_data = train_data_C[i * num_val_samples: (i + 1) * num_val_samples]
    val_targets = train_targets_C[i * num_val_samples: (i + 1) * num_val_samples]

    # 훈련 데이터 준비: 다른 분할 전체
    partial_train_data = np.concatenate(
        [train_data_C[:i * num_val_samples],
         train_data_C[(i + 1) * num_val_samples:]],
        axis=0)
    partial_train_targets = np.concatenate(
        [train_targets_C[:i * num_val_samples],
         train_targets_C[(i + 1) * num_val_samples:]],
        axis=0)

    # 케라스 모델 구성(컴파일 포함)
    model_C = build_model_C()
    # 모델 훈련(verbose=0 이므로 훈련 과정이 출력되지 않습니다)
    history = model_C.fit(partial_train_data, partial_train_targets,
                        validation_data=(val_data, val_targets),
                        epochs=num_epochs, batch_size=1, verbose=0)
    mae_history = history.history['val_mean_absolute_error']
    all_mae_histories_C.append(mae_history)
# not cure

num_epochs = 1000
all_mae_historiesNC = []
for i in range(k):
    print('처리중인 폴드 #', i)
    # 검증 데이터 준비: k번째 분할
    val_data = train_dataNC[i * num_val_samples: (i + 1) * num_val_samples]
    val_targets = train_targetsNC[i * num_val_samples: (i + 1) * num_val_samples]

    # 훈련 데이터 준비: 다른 분할 전체
    partial_train_data = np.concatenate(
        [train_dataNC[:i * num_val_samples],
         train_dataNC[(i + 1) * num_val_samples:]],
        axis=0)
    partial_train_targets = np.concatenate(
        [train_targetsNC[:i * num_val_samples],
         train_targetsNC[(i + 1) * num_val_samples:]],
        axis=0)

    # 케라스 모델 구성(컴파일 포함)
    modelNC = build_modelNC()
    # 모델 훈련(verbose=0 이므로 훈련 과정이 출력되지 않습니다)
    history = modelNC.fit(partial_train_data, partial_train_targets,
                        validation_data=(val_data, val_targets),
                        epochs=num_epochs, batch_size=1, verbose=0)
    mae_history = history.history['val_mean_absolute_error']
    all_mae_historiesNC.append(mae_history)
average_mae_history_C = [
    np.mean([x[i] for x in all_mae_histories_C]) for i in range(num_epochs)]

average_mae_historyNC = [
    np.mean([x[i] for x in all_mae_historiesNC]) for i in range(num_epochs)]
import matplotlib.pyplot as plt
plt.plot(range(1, len(average_mae_history_C) + 1), average_mae_history_C)
plt.title('cure')
plt.xlabel('Epochs')
plt.ylabel('Validation MAE')
plt.show()
plt.plot(range(1, len(average_mae_historyNC) + 1), average_mae_historyNC)
plt.title('not cure')
plt.xlabel('Epochs')
plt.ylabel('Validation MAE')
plt.show()
def smooth_curve(points, factor=0.9):
  smoothed_points = []
  for point in points:
    if smoothed_points:
      previous = smoothed_points[-1]
      smoothed_points.append(previous * factor + point * (1 - factor))
    else:
      smoothed_points.append(point)
  return smoothed_points

smooth_mae_history_C = smooth_curve(average_mae_history_C[10:])

plt.plot(range(1, len(smooth_mae_history_C) + 1), smooth_mae_history_C)
plt.title('cure')
plt.xlabel('Epochs')
plt.ylabel('Validation MAE')
plt.show()
def smooth_curve(points, factor=0.9):
  smoothed_points = []
  for point in points:
    if smoothed_points:
      previous = smoothed_points[-1]
      smoothed_points.append(previous * factor + point * (1 - factor))
    else:
      smoothed_points.append(point)
  return smoothed_points

smooth_mae_historyNC = smooth_curve(average_mae_historyNC[10:])

plt.plot(range(1, len(smooth_mae_historyNC) + 1), smooth_mae_historyNC)
plt.title('not cure')
plt.xlabel('Epochs')
plt.ylabel('Validation MAE')
plt.show()
pred_targets=model_C.predict(test_data_C)

plt.scatter(pred_targets,test_targets_C)
plt.plot(np.arange(80),np.arange(80))<jupyter_output><empty_output><jupyter_text>선형회귀<jupyter_code>from sklearn.linear_model import LinearRegression

model = LinearRegression()
model.fit(pred_targets, test_targets_C)

model.coef_, model.intercept_ 
xs = np.arange(0, 80, 0.1)
ys = xs*model.coef_[0] + model.intercept_

plt.scatter(pred_targets,test_targets_C)
plt.plot(xs,ys,'r-',lw=3)<jupyter_output><empty_output><jupyter_text>2차식 회귀<jupyter_code>X = np.c_[pred_targets, pred_targets**2]
y = test_targets_C

model = LinearRegression()
model.fit(X, y)

model.coef_, model.intercept_
xs = np.arange(0,80,0.1)
ys = xs*model.coef_[0] + (xs**2)*model.coef_[1] + model.intercept_

plt.scatter(pred_targets,test_targets_C)
plt.plot(xs,ys,'r-',lw=3)
pred_targets=modelNC.predict(test_data_C)

plt.scatter(pred_targets,test_targetsNC)
plt.plot(np.arange(80),np.arange(80))
test_data_C.shape
train_data_C.shape
test_data_C[0]
time_C=model_C.predict(test_data_C)[0]
timeNC=modelNC.predict(test_data_C)[0]

if time_C > timeNC:
  print('치료하세용')
else:
  print('치료하지 마세용')

print(test_targets_C[0],time_C,timeNC,time_C-timeNC)


# import lightgbm
import tensorflow as tf
# 2. 모델의 구조를 만듭니다
X = tf.keras.layers.Input(shape=[13])
H = tf.keras.layers.Dense(8, activation='swish')(X)
H = tf.keras.layers.Dense(8, activation='swish')(H)
H = tf.keras.layers.Dense(8, activation='swish')(H)
Y = tf.keras.layers.Dense(1)(H)
model = tf.keras.models.Model(X, Y)
model.compile(loss='mse')
# 2. 모델의 구조를 BatchNormalization layer를 사용하여 만든다.
X = tf.keras.layers.Input(shape=[17])

H = tf.keras.layers.Dense(16)(X)
H = tf.keras.layers.BatchNormalization()(H)
H = tf.keras.layers.Activation('swish')(H)

H = tf.keras.layers.Dense(8)(H)
H = tf.keras.layers.BatchNormalization()(H)
H = tf.keras.layers.Activation('swish')(H)

H = tf.keras.layers.Dense(8)(H)
H = tf.keras.layers.BatchNormalization()(H)
H = tf.keras.layers.Activation('swish')(H)

Y = tf.keras.layers.Dense(1)(H)
model = tf.keras.models.Model(X, Y)
model.compile(loss='mse')
# 3.데이터로 모델을 학습(FIT)합니다.
model.fit(train_data_C, train_targets_C, epochs=1000)
# 4. 모델을 이용합니다
print(model.predict(test_data_C))
# 종속변수 확인
print(test_targets_C)
# 환자 특성 분포
plt.plot(data_CX, 'o')
dataX[np.where((dataA == 1) & (dataY <= 100))[0]]
# 치료받은 사람들 중 생존시간이 100이하인 사람들만 고른다.
dataCX_100 = dataX[np.where((dataA == 1) & (dataY <= 100))[0]]
dataCY_100 = dataY[np.where((dataA == 1) & (dataY <= 100))[0]]

# 치료받지 않은 사람들 중 생존시간이 100이하인 사람들만 고른다.
dataNCX_100 = dataX[np.where((dataA == 0) & (dataY <= 100))[0]]
dataNCY_100 = dataY[np.where((dataA == 0) & (dataY <= 100))[0]]
import tensorflow as tf

###########################
# 2. 모델의 구조를 만듭니다
X = tf.keras.layers.Input(shape=[17])
H = tf.keras.layers.Dense(8, activation='swish')(X)
H = tf.keras.layers.Dense(8, activation='swish')(H)
H = tf.keras.layers.Dense(8, activation='swish')(H)
Y = tf.keras.layers.Dense(1)(H)
model = tf.keras.models.Model(X, Y)
model.compile(loss='mse')
 
# 2. 모델의 구조를 BatchNormalization layer를 사용하여 만든다.
X = tf.keras.layers.Input(shape=[17])
 
H = tf.keras.layers.Dense(8)(X)
H = tf.keras.layers.BatchNormalization()(H)
H = tf.keras.layers.Activation('swish')(H)
 
H = tf.keras.layers.Dense(8)(H)
H = tf.keras.layers.BatchNormalization()(H)
H = tf.keras.layers.Activation('swish')(H)
 
H = tf.keras.layers.Dense(8)(H)
H = tf.keras.layers.BatchNormalization()(H)
H = tf.keras.layers.Activation('swish')(H)
 
Y = tf.keras.layers.Dense(1)(H)
model = tf.keras.models.Model(X, Y)
model.compile(loss='mse')
train_data_C,test_data_C,train_targets_C,test_targets_C=train_test_split(dataCX_100,dataCY_100)
train_dataNC,test_dataNC,train_targetsNC,test_targetsNC=train_test_split(dataNCX_100,dataNCY_100)
# 3.데이터로 모델을 학습(FIT)합니다.
# model = build_model_C_100()
model = build_model_C_100_16()
history = model.fit(train_data_C, train_targets_C, epochs=1000)
# 4. 모델을 이용합니다
pred_y = model.predict(test_data_C) 
print(pred_y)
# 종속변수 확인
print(test_targets_C)
# 종속변수 확인
error = test_targets_C - pred_y
print(error)
error.shape
plt.plot(error, 'bo')
mae_C = tf.keras.losses.MAE(test_targets_C, pred_y)
mae_C.shape
# 생존 시간 오차가 10 이하인 개수
np.sum(np.where(mae_C <= 10, 1, 0))
# 생존 시간 오차가 10 초과인 개수
np.sum(np.where(mae_C > 10, 1, 0))
test_dataNC.shape
train_dataNC.shape
train_targetsNC.shape
# 3.치료하지 않았고 생존시간이 100 이하인 환자 데이터로 모델을 학습(FIT)합니다.
# model_NC = build_model_NC_100()
model_NC = build_model_NC_100_16()
model_NC.fit(train_dataNC, train_targetsNC, epochs=1000)
pred_y_NC = model_NC.predict(test_dataNC)
mae_NC = tf.keras.losses.MAE(pred_y_NC, test_targetsNC)
mae_NC.shape
plt.plot(mae_NC, 'ro')
pred_y[0] > pred_y_NC[0]
train_data_C.shape
type(mae_NC)
mae_NC[0]
# 오차가 10 이하인 개수
np.sum(np.where(mae_NC <= 10, 1, 0))
# 오차가 10 초과인 개수
np.sum(np.where(mae_NC > 10, 1, 0))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14928198433420364, 'fraction_numerical': 0.02147519582245431, 'mean_word_length': 4.42528328611898, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '141743', 'license': 'MIT', 'n_tokens_mistral': 7203, 'n_tokens_neox': 7318, 'n_words': 1046, 'path': 'DHH_127page_20200824.ipynb', 'repo_name': 'brothergive/dhh_2020', 'size': '15320'}