<jupyter_start><jupyter_text>Explore for 11 epochs (epoch + 5 minutes before and 5 minutes after)<jupyter_code>#Explore for 11 epochs (epoch + 5 minutes before and 5 minutes after)
i=11
train5 = reshape_n(train.iloc[:,:-1], i)
train5["Y"]= train.iloc[:,-1]

for j in range(0, i//2):
  train5.drop([train5.shape[0]-(i+1)], inplace=True)

test5 = reshape_n(test.iloc[:,:-1], i)
test5["Y"]= test.iloc[:,-1].values
for j in range(0, i//2):
  test5.drop([test5.shape[0]-(i+1)], inplace=True)

X = train5.iloc[:,:-1]
y = train5.iloc[:,-1]
X_test = test5.iloc[:,:-1]
y_true = test5.iloc[:,-1]
print(X.shape)
print(y.shape)
print(y.unique())
Around best score: n_estimators = 180, regularization 5
%%time
errors = []
Lk = []
La = []

r = [5, 8, 9, 10, 11, 12, 13, 14, 15]

for i in r:
    xbc = xgb.XGBClassifier(n_estimators = 180, random_state=42, learning_rate= 0.1, max_depth= 8,  subsample= 0.7, 
                            n_jobs=-2, reg_lambda=i)
    xbc.fit(X, y)
    ll = log_loss(y_true, xbc.predict_proba(X_test))
    errors.append(ll)
    y_pred = xbc.predict(X_test)
    k=cohen_kappa_score(y_true, y_pred)
    a= accuracy_score(y_true, y_pred)
    print("for n_estimators= ", i)
    print("log loss = ", ll)
    print("kappa = ", k)
    print("accuracy = ", a)
    
    Lk.append(k)
    La.append(a)
plt.plot(r, errors, label = "log loss")
plt.legend(loc='lower right')
plt.show()

plt.plot(r, Lk, label = "kappa")
plt.legend(loc='lower right')
plt.show()<jupyter_output><empty_output><jupyter_text>Now on n_estimators<jupyter_code>%%time
errors = []
Lk = []
La = []

r = range(170, 190, 1)

for i in r:
    xbc = xgb.XGBClassifier(n_estimators = i, random_state=42, learning_rate= 0.1, max_depth= 8,  subsample= 0.7, 
                            n_jobs=-2, reg_lambda=10)
    xbc.fit(X, y)
    ll = log_loss(y_true, xbc.predict_proba(X_test))
    errors.append(ll)
    y_pred = xbc.predict(X_test)
    k=cohen_kappa_score(y_true, y_pred)
    a= accuracy_score(y_true, y_pred)
    print("for n_estimators= ", i)
    print("log loss = ", ll)
    print("kappa = ", k)
    print("accuracy = ", a)
    
    Lk.append(k)
    La.append(a)
plt.plot(r, errors, label = "log loss")
plt.legend(loc='lower right')
plt.show()

plt.plot(r, Lk, label = "kappa")
plt.legend(loc='lower right')
plt.show()
%%time
errors = []
Lk = []
La = []

r = [5, 10, 14, 15]

for i in r:
    xbc = xgb.XGBClassifier(n_estimators = 178, random_state=42, learning_rate= 0.1, max_depth= 8,  subsample= 0.7, 
                            n_jobs=-2, reg_lambda=i)
    xbc.fit(X, y)
    ll = log_loss(y_true, xbc.predict_proba(X_test))
    errors.append(ll)
    y_pred = xbc.predict(X_test)
    k=cohen_kappa_score(y_true, y_pred)
    a= accuracy_score(y_true, y_pred)
    print("for reg_lambda= ", i)
    print("log loss = ", ll)
    print("kappa = ", k)
    print("accuracy = ", a)
    
    Lk.append(k)
    La.append(a)
plt.plot(r, errors, label = "log loss")
plt.legend(loc='lower right')
plt.show()

plt.plot(r, Lk, label = "kappa")
plt.legend(loc='lower right')
plt.show()<jupyter_output><empty_output><jupyter_text>Resuming old thread<jupyter_code>errors = []
Lk = []
La = []

r = range(10, 500 , 10)

for i in r:
    xbc = xgb.XGBClassifier(n_estimators = i, random_state=42, learning_rate= 0.1, max_depth= 8,  subsample= 0.7, 
                            n_jobs=-2, warm_start=True)
    xbc.fit(X, y)
    ll = log_loss(y_true, xbc.predict_proba(X_test))
    errors.append(ll)
    y_pred = xbc.predict(X_test)
    k=cohen_kappa_score(y_true, y_pred)
    a= accuracy_score(y_true, y_pred)
    print("for n_estimators= ", i)
    print("log loss = ", ll)
    print("kappa = ", k)
    print("accuracy = ", a)
    
    Lk.append(k)
    La.append(a)
    xbc.fit(X, y).n_estimators += 10
rr = range(10, 460 , 10)
plt.plot(rr, errors, label = "log loss")
plt.legend(loc='lower right')
plt.show()
#plt.plot(rr, La, label = "accuracy")
plt.plot(rr, Lk, label = "kappa")
plt.legend(loc='lower right')
plt.show()<jupyter_output><empty_output><jupyter_text>n=100. Score: 0.692 compared to 0.691 with N=5- log loss =  0.6070592229998082- kappa =  0.7129564405419131- accuracy =  0.7888368907658942log loss minimized for n_estimators=  110- log loss =  0.6066953947836593- kappa =  0.7144819405077073- accuracy =  0.78997831297797051st max kappa- for n_estimators=  270- log loss =  0.66123282243177- kappa =  0.7278202163159542- accuracy =  0.7997945440018263let's see if regularisation help for n = 270<jupyter_code>errors = []
Lk = []
La = []

r = range(120, 370 , 50)
xbc = xgb.XGBClassifier(n_estimators = 120, random_state=42, learning_rate= 0.1, max_depth= 8,  subsample= 0.7, 
                            n_jobs=-2, warm_start=True, lambda_reg=5)

for i in r:
    xbc.fit(X, y)
    ll = log_loss(y_true, xbc.predict_proba(X_test))
    errors.append(ll)
    y_pred = xbc.predict(X_test)
    k=cohen_kappa_score(y_true, y_pred)
    a= accuracy_score(y_true, y_pred)
    print("for n_estimators= ", i)
    print("log loss = ", ll)
    print("kappa = ", k)
    print("accuracy = ", a)
    
    Lk.append(k)
    La.append(a)
    xbc.fit(X, y).n_estimators += 50
plt.plot(r, errors, label = "log loss")
plt.legend(loc='lower right')
plt.show()
#plt.plot(r, La, label = "accuracy")
plt.plot(r, Lk, label = "kappa")
plt.legend(loc='lower right')
plt.show()
xbc = xgb.XGBClassifier(n_estimators = 180, random_state=42, max_depth=8, learning_rate=0.1, subsample=0.7)

parametres = { 'reg_lambda' : [0, 1, 5], 'gamma': [0, 0.01], 'reg_alpha' : [0, 1, 5]}
    
ck_score = make_scorer(cohen_kappa_score)
grid = GridSearchCV(estimator=xbc, param_grid=parametres, scoring='neg_log_loss', n_jobs=-1, verbose=2)

grid_fitted = grid.fit(X,y)
print(grid_fitted.best_params_)

y_pred = grid.predict(X_test)

print("kappa: ", cohen_kappa_score(y_true, y_pred))
print("accurancy for n_estimators  = " , accuracy_score(y_true, y_pred))
ll = log_loss(y_true, grid.predict_proba(X_test))
ll<jupyter_output><empty_output><jupyter_text>Reference without regularization n: 180- log loss =  0.632- kappa =  0.722- accuracy =  0.795n estimator : 180, {'gamma': 0.01, 'reg_alpha': 5, 'reg_lambda': 5}- log loss: 0.585- kappa:  0.711- accurancy: 0.787for n_estimators=  240 {'gamma': 0.01, 'reg_alpha': 5, 'reg_lambda': 5} // log loss min - log loss =  0.5805557190155572- kappa =  0.719 /  - accuracy =  0.792- challenge score : 0.69501856567217. Delta test/Challenge 0,7186789415933239- 0,69501856567217= 0.024for n_estimators=  300 {'gamma': 0.01, 'reg_alpha': 5, 'reg_lambda': 5} // point d'interret- log loss =  0.580970969589507- kappa =  0.7218863968707475- accuracy =  0.7947722862686908for n_estimators=  340 {'gamma': 0.01, 'reg_alpha': 5, 'reg_lambda': 5} // point d'interret- log loss =  0.5826844474036029- kappa =  0.7242661136765393- accuracy =  0.7965985618080128- Your score is : 0.69083908105379. Moins que n = 240 faut arreter la.<jupyter_code>errors = []
Lk = []
La = []

r = range(100, 420 , 20)

xbc = xgb.XGBClassifier(n_estimators = 100, random_state=42, learning_rate= 0.1, max_depth= 8,  subsample= 0.7, 
                            n_jobs=-2, warm_start=True, reg_lambda=5, reg_alpha=5, gamma=0.01)
for i in r:

    xbc.fit(X, y)
    ll = log_loss(y_true, xbc.predict_proba(X_test))
    errors.append(ll)
    y_pred = xbc.predict(X_test)
    k=cohen_kappa_score(y_true, y_pred)
    a= accuracy_score(y_true, y_pred)
    print("for n_estimators= ", i)
    print("log loss = ", ll)
    print("kappa = ", k)
    print("accuracy = ", a)
    
    Lk.append(k)
    La.append(a)
    xbc.fit(X, y).n_estimators += 20
plt.plot(r, errors, label = "log loss")
plt.legend(loc='lower right')
plt.show()
#plt.plot(r, La, label = "accuracy")
plt.plot(r, Lk, label = "kappa")
plt.legend(loc='lower right')
plt.show()<jupyter_output><empty_output><jupyter_text>trying with 17 epochs (to make sure)- submitted for estimators = 150 | 180 with score a bit below 0.70<jupyter_code>#Explore for 11 epochs (epoch + 5 minutes before and 5 minutes after)
i=17
train5 = reshape_n(train.iloc[:,:-1], i)
train5["Y"]= train.iloc[:,-1]

for j in range(0, i//2):
  train5.drop([train5.shape[0]-(i+1)], inplace=True)

test5 = reshape_n(test.iloc[:,:-1], i)
test5["Y"]= test.iloc[:,-1].values
for j in range(0, i//2):
  test5.drop([test5.shape[0]-(i+1)], inplace=True)

X = train5.iloc[:,:-1]
y = train5.iloc[:,-1]
X_test = test5.iloc[:,:-1]
y_true = test5.iloc[:,-1]
X.shape
errors = []
Lk = []
La = []

r = range(10, 210 , 10)


for i in r:
    xbc = xgb.XGBClassifier(n_estimators = i, random_state=42, learning_rate= 0.1, max_depth= 8,  subsample= 0.7, 
                            n_jobs=-1, warm_start=True, lambda_reg=5)
    xbc.fit(X, y)
    ll = log_loss(y_true, xbc.predict_proba(X_test))
    errors.append(ll)
    y_pred = xbc.predict(X_test)
    k=cohen_kappa_score(y_true, y_pred)
    a= accuracy_score(y_true, y_pred)
    print("for n_estimators= ", i)
    print("log loss = ", ll)
    print("kappa = ", k)
    print("accuracy = ", a)
    
    Lk.append(k)
    La.append(a)
    


plt.plot(r, errors, label = "log loss")
plt.legend(loc='lower right')
plt.show()
#plt.plot(r, La, label = "accuracy")
plt.plot(r, Lk, label = "kappa")
plt.legend(loc='lower right')
plt.show()<jupyter_output><empty_output><jupyter_text>Illustrating best score  your score 0.70273772415489xbc = xgb.XGBClassifier(n_estimators = 180, random_state=42, learning_rate= 0.1, max_depth= 8,  subsample= 0.7,                             n_jobs=-2, reg_lambda=5)<jupyter_code>#Explore for 11 epochs (epoch + 5 minutes before and 5 minutes after)
i=11
train5 = reshape_n(train.iloc[:,:-1], i)
train5["Y"]= train.iloc[:,-1]

for j in range(0, i//2):
  train5.drop([train5.shape[0]-(i+1)], inplace=True)

test5 = reshape_n(test.iloc[:,:-1], i)
test5["Y"]= test.iloc[:,-1].values
for j in range(0, i//2):
  test5.drop([test5.shape[0]-(i+1)], inplace=True)

X = train5.iloc[:,:-1]
y = train5.iloc[:,-1]
X_test = test5.iloc[:,:-1]
y_true = test5.iloc[:,-1]
xbc = xgb.XGBClassifier(n_estimators = 180, random_state=42, learning_rate= 0.1, max_depth= 8,  subsample= 0.7, 
                            n_jobs=-2, reg_lambda=5)

xbc.fit(X, y)
ll = log_loss(y_true, xbc.predict_proba(X_test))
errors.append(ll)
y_pred = xbc.predict(X_test)
k=cohen_kappa_score(y_true, y_pred)
a= accuracy_score(y_true, y_pred)
print("for n_estimators= ", i)
print("log loss = ", ll)
print("kappa = ", k)
print("accuracy = ", a)
def plot_confusion_matrix(cm, classes,
                          normalize=False,
                          title='Confusion matrix',
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')

    print(cm)

    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    fmt = '.2f' if normalize else 'd'
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt),
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')

# Compute confusion matrix
cnf_matrix = confusion_matrix(y_true, y_pred)
np.set_printoptions(precision=2)

# Plot non-normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix, classes=[0, 1, 2, 3, 4], title='Confusion matrix, without normalization')

# Plot normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix, classes=[0, 1, 2, 3, 4], normalize=True,
                      title='Normalized confusion matrix')

plt.show()
lim = y_pred.shape[0]

fig, ax = plt.subplots(figsize=(20,5))
plt.plot(y_true)
plt.ylabel('Sleep stage')
plt.yticks([0, 1, 2, 3, 4, ])
ax.set_xlim([0, lim])
fig.suptitle("Original manualy scored hypnogram")
plt.show()


fig, ax = plt.subplots(figsize=(20,5))
plt.plot(y_pred)
plt.ylabel('Sleep stage')
plt.yticks([0, 1, 2, 3, 4, ])
ax.set_xlim([0, lim])
fig.suptitle("Estimated hypnogram")
plt.show()<jupyter_output><empty_output><jupyter_text>Trying stacking reference boostingfor n_estimators=  11log loss =  0.6064331996516102kappa =  0.719398312246695accuracy =  0.7934025796141992XGboost on XGboost: exactly the same challenge score<jupyter_code>naif = pd.read_excel('C:\\Users\\i053131\\Desktop\\Epilepsie\\Dreem\\data\\interim\\featuresTrain.xlsx')
eeg = pd.read_excel('C:\\Users\\i053131\\Desktop\\Epilepsie\\Dreem\\data\\interim\\spectrogram_eeg_features30Train.xlsx')
acc = pd.read_excel('C:\\Users\\i053131\\Desktop\\Epilepsie\\Dreem\\data\\interim\\acc_featuresTrain.xlsx')
pulse = pd.read_csv('C:\\Users\\i053131\\Desktop\\Epilepsie\\Dreem\\data\\interim\\pulse_featuresTrain.csv')
pulse = pulse.iloc[:, 1:]  
eeg.drop(columns=["eeg1_Above100Hz0", "eeg2_Above100Hz0", "eeg3_Above100Hz0", "eeg4_Above100Hz0"], inplace=True)
df = pd.concat([eeg, acc, pulse, naif], axis=1)

train = df.iloc[0:int(df.shape[0]*0.8), :]
test = df.iloc[int(df.shape[0]*0.8):, :]

i=11
train5 = reshape_n(train.iloc[:,:-1], i)
train5["Y"]= train.iloc[:,-1]

for j in range(0, i//2):
  train5.drop([train5.shape[0]-(i+1)], inplace=True)

test5 = reshape_n(test.iloc[:,:-1], i)
test5["Y"]= test.iloc[:,-1].values
for j in range(0, i//2):
  test5.drop([test5.shape[0]-(i+1)], inplace=True)

X = train5.iloc[:,:-1]

X_test = test5.iloc[:,:-1]
y_true = test5.iloc[:,-1]
y = train5.iloc[:,-1]
y_true = test5.iloc[:,-1]
y.isna().any()
xbc = xgb.XGBClassifier(n_estimators = 180, random_state=42, learning_rate= 0.1, max_depth= 8,  subsample= 0.7, 
                            n_jobs=-2, reg_lambda=5)
xbc.fit(X, y)

proba_X_test = xbc.predict_proba(X_test)
proba_X = xbc.predict_proba(X)
ll = log_loss(y_true, proba_X_test)

y_pred = xbc.predict(X_test)
k=cohen_kappa_score(y_true, y_pred)
a= accuracy_score(y_true, y_pred)
print("for n_estimators= ", i)
print("log loss = ", ll)
print("kappa = ", k)
print("accuracy = ", a)
tmp = pd.DataFrame(proba_X)
i=11
train_proba11 = reshape_n(tmp, i)
train_proba11["Y"]= y.values

for j in range(0, i//2):
  train_proba11.drop([train_proba11.shape[0]-(i+1)], inplace=True)
tmp2 = pd.DataFrame(proba_X_test)
i=11
test_proba11 = reshape_n(tmp2, i)
test_proba11["Y"]= y_true.values
for j in range(0, i//2):
  test_proba11.drop([test_proba11.shape[0]-(i+1)], inplace=True)


train_proba11.shape
train_proba11["Y"].shape
train_proba11.to_excel('C:\\Users\\i053131\\Desktop\\Epilepsie\\Dreem\\data\\interim\\proba11train.xlsx')
test_proba11.to_excel('C:\\Users\\i053131\\Desktop\\Epilepsie\\Dreem\\data\\interim\\proba11test.xlsx')
X = train_proba11.iloc[:,:-1]
y = train_proba11.iloc[:,-1]
X_test = test_proba11.iloc[:,:-1]
y_true = test_proba11.iloc[:,-1]
xbcs = xgb.XGBClassifier(n_estimators = 17, random_state=42, learning_rate= 0.1, max_depth= 2,  subsample= 0.7, 
                            n_jobs=-2)
xbcs.fit(X, y)
y_pred = xbcs.predict(X_test)
k=cohen_kappa_score(y_true, y_pred)
a= accuracy_score(y_true, y_pred)
print("for n_estimators= 17")

print("kappa = ", k)
print("accuracy = ", a)
errors = []
Lk = []
La = []

r = range(10, 21)


for i in r:
    xbcs = xgb.XGBClassifier(n_estimators = i, random_state=42, learning_rate= 0.1, max_depth= 2,  subsample= 0.7, 
                            n_jobs=-1)
    xbcs.fit(train_proba11, y)
    ll = log_loss(y_true, xbcs.predict_proba(test_proba11))
    errors.append(ll)
    y_pred = xbcs.predict(test_proba11)
    k=cohen_kappa_score(y_true, y_pred)
    a= accuracy_score(y_true, y_pred)
    print("n_estimators= ", i)
    print("log loss = ", ll)
    print("kappa = ", k)
    print("accuracy = ", a)
    
    Lk.append(k)
    La.append(a)
plt.plot(r, errors, label = "log loss")
plt.legend(loc='upper right')
plt.show()
#plt.plot(r, La, label = "accuracy")
plt.plot(r, Lk, label = "kappa")
plt.legend(loc='lower right')
plt.show()
errors = []
Lk = []
La = []

r = range(1, 5)


for i in r:
    xbcs = xgb.XGBClassifier(n_estimators = 17, random_state=42, learning_rate= 0.1, max_depth= i,  subsample= 0.7, 
                            n_jobs=-1)
    xbcs.fit(train_proba11, y)
    ll = log_loss(y_true, xbcs.predict_proba(test_proba11))
    errors.append(ll)
    y_pred = xbcs.predict(test_proba11)
    k=cohen_kappa_score(y_true, y_pred)
    a= accuracy_score(y_true, y_pred)
    print("depth= ", i)
    print("log loss = ", ll)
    print("kappa = ", k)
    print("accuracy = ", a)
    
    Lk.append(k)
    La.append(a)
plt.plot(r, errors, label = "log loss")
plt.legend(loc='lower right')
plt.show()
#plt.plot(r, La, label = "accuracy")
plt.plot(r, Lk, label = "kappa")
plt.legend(loc='lower right')
plt.show()
errors = []
Lk = []
La = []

r = range(1, 11)


for i in r:
    xbcs = xgb.XGBClassifier(n_estimators = 17, random_state=42, learning_rate= 0.1, max_depth= 2,  subsample= i/10, 
                            n_jobs=-1)
    xbcs.fit(train_proba11, y)
    ll = log_loss(y_true, xbcs.predict_proba(test_proba11))
    errors.append(ll)
    y_pred = xbcs.predict(test_proba11)
    k=cohen_kappa_score(y_true, y_pred)
    a= accuracy_score(y_true, y_pred)
    print("subsample ", i/10)
    print("log loss = ", ll)
    print("kappa = ", k)
    print("accuracy = ", a)
    
    Lk.append(k)
    La.append(a)
plt.plot(r, errors, label = "log loss")
plt.legend(loc='lower right')
plt.show()
#plt.plot(r, La, label = "accuracy")
plt.plot(r, Lk, label = "kappa")
plt.legend(loc='lower right')
plt.show()<jupyter_output><empty_output><jupyter_text>let's try by stacking logistic regression<jupyter_code>train_proba11.shape
def predwake(X, y, X_test, y_true, L=[0.001, 0.01, 0.1, 1, 10, 100]):
    Lkappa_l1 = []
    Lkappa_l2 = []
    Lsparsity_l1 = []
    Lsparsity_l2 = []
    Lrecall_l1 = []
    Lrecall_l2 = []
    Lprecision_l1 = []
    Lprecision_l2 = []
    for C in L: 
        l1_LR = LogisticRegression(C=C, penalty='l1', tol=0.01, solver='liblinear')
        l2_LR = LogisticRegression(C=C, penalty='l2', tol=0.01, solver='liblinear')
        l1_LR.fit(X, y)
        l2_LR.fit(X, y)
        y_pred_l1 = l1_LR.predict(X_test)
        y_pred_l2 = l2_LR.predict(X_test)
        coef_l1_LR = l1_LR.coef_.ravel()
        coef_l2_LR = l2_LR.coef_.ravel()
        sparsity_l1_LR = np.mean(coef_l1_LR == 0) * 100
        sparsity_l2_LR = np.mean(coef_l2_LR == 0) * 100
        kappa_l1 = cohen_kappa_score(y_true, y_pred_l1)
        kappa_l2 = cohen_kappa_score(y_true, y_pred_l2)
        #
        Lsparsity_l1.append(sparsity_l1_LR)
        Lsparsity_l2.append(sparsity_l2_LR)

        Lkappa_l1.append(kappa_l1)
        Lkappa_l2.append(kappa_l2)
 
#        notScarseCoefs = []
#        for i in range(0, len(coef_l1_LR)):
#            if (coef_l1_LR[i] !=0):
#                notScarseCoefs.append(X.columns[i])
    fig, ax = plt.subplots()
    ax.set_title("Sparsity vs C for l1")
    ax.semilogx(L, Lsparsity_l1)
    plt.show()
    fig, ax = plt.subplots()
    ax.set_title("Sparsity vs C for l2")
    ax.semilogx(L, Lsparsity_l2)
    plt.show()
    
               
    fig, ax = plt.subplots()
    ax.set_title("kappa vs C for l1 (blue') and l2 (red)")
    ax.semilogx(L, Lkappa_l1, color= 'b')
    #plt.legend(loc='upper right')
    ax.semilogx(L, Lkappa_l2, color = 'r')
    plt.show()
    return [Lkappa_l1, Lkappa_l2]
predwake(X, y, X_test, y_true)

#predwake(X, y, X_test, y_true)
L = list(pd.Series(range(70, 120))/10000)
Lkappa_l2 = []

for C in L: 
    
    l2_LR = LogisticRegression(C=C, penalty='l2', tol=0.01, solver='liblinear')
 
    l2_LR.fit(X, y)
    y_pred_l2 = l2_LR.predict(X_test)


    kappa_l2 = cohen_kappa_score(y_true, y_pred_l2)
    Lkappa_l2.append(kappa_l2)


plt.plot(L, Lkappa_l2, label = "kappa")
plt.legend(loc='lower right')
plt.show()

plt.plot(L, Lkappa_l2, label = "kappa")
plt.legend(loc='lower right')
plt.show()
l2_LR = LogisticRegression(C=0.009, penalty='l2', tol=0.01, solver='liblinear')
l2_LR.fit(train_proba11, y)
y_pred_l2 = l2_LR.predict(test_proba11)
kappa_l2 = cohen_kappa_score(y_true, y_pred_l2)
print(kappa_l2)<jupyter_output>0.7295425544577583<jupyter_text>with Total energyReference one 1 epoch (previous)- log loss =  0.7206771579062425- kappa =  0.6518430793293312- accuracy =  0.743668720054757with (summ of) total energies: a bit better than with amp but barelylog loss =  0.7278049343346922kappa =  0.6525825770729099with energy by captors- log loss =  0.7164148313083308- kappa =  0.6519482657334398<jupyter_code>trainT = dfT.iloc[0:int(df.shape[0]*0.8), :]
testT = dfT.iloc[int(df.shape[0]*0.8):, :]
%%time
errors = []
Lk = []
La = []

X = trainT.iloc[:,:-1]
y = trainT.iloc[:,-1]
print(y.unique())
X_test = testT.iloc[:,:-1]
y_true = testT.iloc[:,-1]
xbc = xgb.XGBClassifier(n_estimators = 170, random_state=42, learning_rate= 0.1, max_depth= 8,  subsample= 0.7, 
                        n_jobs=-2)
xbc.fit(X, y)
ll = log_loss(y_true, xbc.predict_proba(X_test))
errors.append(ll)
y_pred = xbc.predict(X_test)
k=cohen_kappa_score(y_true, y_pred)
a= accuracy_score(y_true, y_pred)
print("considering only one epoch")
print("log loss = ", ll)
print("kappa = ", k)
print("accuracy = ", a)
Lk.append(k)
La.append(a)<jupyter_output>[0 1 2 3 4]
considering only one epoch
log loss =  0.7164148313083308
kappa =  0.6519482657334398
accuracy =  0.743668720054757
Wall time: 2min 55s<jupyter_text>add pulse amplitude, discard "double features<jupyter_code>trainA = dfa.iloc[0:int(df.shape[0]*0.8), :]
testA = dfa.iloc[int(df.shape[0]*0.8):, :]
Reference one 1 epoch (previous)
- log loss =  0.7206771579062425
- kappa =  0.6518430793293312
- accuracy =  0.743668720054757

now
- log loss =  0.7030750665966249
- kappa =  0.6597300972384683
- accuracy =  0.7500570385580653

(tiny bit better)<jupyter_output><empty_output><jupyter_text>reference 1 epoch<jupyter_code>train = df.iloc[0:int(df.shape[0]*0.8), :]
test = df.iloc[int(df.shape[0]*0.8):, :]
errors = []
Lk = []
La = []

X = train.iloc[:,:-1]
y = train.iloc[:,-1]
X_test = test.iloc[:,:-1]
y_true = test.iloc[:,-1]
xbc = xgb.XGBClassifier(n_estimators = 170, random_state=42, learning_rate= 0.1, max_depth= 8,  subsample= 0.7, 
                        n_jobs=-2)
xbc.fit(X, y)
ll = log_loss(y_true, xbc.predict_proba(X_test))
errors.append(ll)
y_pred = xbc.predict(X_test)
k=cohen_kappa_score(y_true, y_pred)
a= accuracy_score(y_true, y_pred)
print("considering only one epoch")
print("log loss = ", ll)
print("kappa = ", k)
print("accuracy = ", a)
Lk.append(k)
La.append(a)<jupyter_output>considering only one epoch
log loss =  0.7206771579062425
kappa =  0.6518430793293312
accuracy =  0.743668720054757<jupyter_text>with amplitude (and cleaning)<jupyter_code>%%time
errors = []
Lk = []
La = []

X = trainA.iloc[:,:-1]
y = trainA.iloc[:,-1]
print(y.unique())
X_test = testA.iloc[:,:-1]
y_true = testA.iloc[:,-1]
xbc = xgb.XGBClassifier(n_estimators = 170, random_state=42, learning_rate= 0.1, max_depth= 8,  subsample= 0.7, 
                        n_jobs=-2)
xbc.fit(X, y)
ll = log_loss(y_true, xbc.predict_proba(X_test))
errors.append(ll)
y_pred = xbc.predict(X_test)
k=cohen_kappa_score(y_true, y_pred)
a= accuracy_score(y_true, y_pred)
print("considering only one epoch")
print("log loss = ", ll)
print("kappa = ", k)
print("accuracy = ", a)
Lk.append(k)
La.append(a)<jupyter_output>[0 1 2 3 4]
considering only one epoch
log loss =  0.7030750665966249
kappa =  0.6597300972384683
accuracy =  0.7500570385580653
Wall time: 1min 35s<jupyter_text>several epochs reference - 11 epoch with lambda=1: kappa =  0.7217- 11 epoch with lambda=5: 0.7194- new 11 epoch with lambda =5 - 0.7206- new 9 epoch with lambada=1: 0.7244<jupyter_code>%%time
errors = []
Lk = []
La = []

r = [3, 5, 7, 9, 11]

for i in r:
    train5 = reshape_n(train.iloc[:,:-1], i)
    train5["Y"]= train.iloc[:,-1]

    for j in range(0, i//2):
      train5.drop([train5.shape[0]-(i+1)], inplace=True)
    
    test5 = reshape_n(test.iloc[:,:-1], i)
    test5["Y"]= test.iloc[:,-1].values
    for j in range(0, i//2):
      test5.drop([test5.shape[0]-(i+1)], inplace=True)
      
    X = train5.iloc[:,:-1]
    y = train5.iloc[:,-1]
    X_test = test5.iloc[:,:-1]
    y_true = test5.iloc[:,-1]
          
    xbc = xgb.XGBClassifier(n_estimators = 180, random_state=42, learning_rate= 0.1, max_depth= 8,  
                            subsample= 0.7, n_jobs=-2, reg_lambda=5)
    xbc.fit(X, y)
    ll = log_loss(y_true, xbc.predict_proba(X_test))
    errors.append(ll)
    y_pred = xbc.predict(X_test)
    k=cohen_kappa_score(y_true, y_pred)
    a= accuracy_score(y_true, y_pred)
    print("considering ", i, " epochs")
    print("log loss = ", ll)
    print("kappa = ", k)
    print("accuracy = ", a)
    Lk.append(k)
    La.append(a)
plt.plot(r, errors, label = "log loss")
plt.legend(loc='lower right')
plt.show()
#plt.plot(r, La, label = "accuracy")
plt.plot(r, Lk, label = "kappa")
plt.legend(loc='lower right')
plt.show()<jupyter_output><empty_output><jupyter_text>several epoch new<jupyter_code>errors = []
Lk = []
La = []
r = [3, 5, 7, 9, 11]

for i in r:
    train5 = reshape_n(trainA.iloc[:,:-1], i)
    train5["Y"]= trainA.iloc[:,-1]

    for j in range(0, i//2):
      train5.drop([train5.shape[0]-(i+1)], inplace=True)
    test5 = reshape_n(testA.iloc[:,:-1], i)
    test5["Y"]= testA.iloc[:,-1].values
    for j in range(0, i//2):
      test5.drop([test5.shape[0]-(i+1)], inplace=True)
      
    X = train5.iloc[:,:-1]
    y = train5.iloc[:,-1]
    X_test = test5.iloc[:,:-1]
    y_true = test5.iloc[:,-1]
          
    xbc = xgb.XGBClassifier(n_estimators = 180, random_state=42, learning_rate= 0.1, max_depth= 8,  
                            subsample= 0.7, n_jobs=-2, reg_lambda=5)
    xbc.fit(X, y)
    ll = log_loss(y_true, xbc.predict_proba(X_test))
    errors.append(ll)
    y_pred = xbc.predict(X_test)
    k=cohen_kappa_score(y_true, y_pred)
    a= accuracy_score(y_true, y_pred)
    print("considering ", i, " epochs")
    print("log loss = ", ll)
    print("kappa = ", k)
    print("accuracy = ", a)
    Lk.append(k)
    La.append(a)
plt.plot(r, errors, label = "log loss")
plt.legend(loc='lower right')
plt.show()
#plt.plot(r, La, label = "accuracy")
plt.plot(r, Lk, label = "kappa")
plt.legend(loc='lower right')
plt.show()
# with reg_lambda=1
errors = []
Lk = []
La = []
r = [3, 5, 7, 9, 11]

for i in r:
    train5 = reshape_n(trainA.iloc[:,:-1], i)
    train5["Y"]= trainA.iloc[:,-1]

    for j in range(0, i//2):
      train5.drop([train5.shape[0]-(i+1)], inplace=True)
    test5 = reshape_n(testA.iloc[:,:-1], i)
    test5["Y"]= testA.iloc[:,-1].values
    for j in range(0, i//2):
      test5.drop([test5.shape[0]-(i+1)], inplace=True)
      
    X = train5.iloc[:,:-1]
    y = train5.iloc[:,-1]
    X_test = test5.iloc[:,:-1]
    y_true = test5.iloc[:,-1]
          
    xbc = xgb.XGBClassifier(n_estimators = 180, random_state=42, learning_rate= 0.1, max_depth= 8,  
                            subsample= 0.7, n_jobs=-2, reg_lambda=1)
    xbc.fit(X, y)
    ll = log_loss(y_true, xbc.predict_proba(X_test))
    errors.append(ll)
    y_pred = xbc.predict(X_test)
    k=cohen_kappa_score(y_true, y_pred)
    a= accuracy_score(y_true, y_pred)
    print("considering ", i, " epochs")
    print("log loss = ", ll)
    print("kappa = ", k)
    print("accuracy = ", a)
    Lk.append(k)
    La.append(a)
plt.plot(r, errors, label = "log loss")
plt.legend(loc='lower right')
plt.show()
#plt.plot(r, La, label = "accuracy")
plt.plot(r, Lk, label = "kappa")
plt.legend(loc='lower right')
plt.show()
train.columns[25:]
Clean data no amplitute
errors = []
Lk = []
La = []
r = [3, 5, 7, 9, 11]

for i in r:
    train5 = reshape_n(train.iloc[:,:-1], i)
    train5["Y"]= train.iloc[:,-1]

    for j in range(0, i//2):
      train5.drop([train5.shape[0]-(i+1)], inplace=True)
    test5 = reshape_n(test.iloc[:,:-1], i)
    test5["Y"]= test.iloc[:,-1].values
    for j in range(0, i//2):
      test5.drop([test5.shape[0]-(i+1)], inplace=True)
      
    X = train5.iloc[:,:-1]
    y = train5.iloc[:,-1]
    X_test = test5.iloc[:,:-1]
    y_true = test5.iloc[:,-1]
          
    xbc = xgb.XGBClassifier(n_estimators = 180, random_state=42, learning_rate= 0.1, max_depth= 8,  
                            subsample= 0.7, n_jobs=-2, reg_lambda=1)
    xbc.fit(X, y)
    ll = log_loss(y_true, xbc.predict_proba(X_test))
    errors.append(ll)
    y_pred = xbc.predict(X_test)
    k=cohen_kappa_score(y_true, y_pred)
    a= accuracy_score(y_true, y_pred)
    print("considering ", i, " epochs")
    print("log loss = ", ll)
    print("kappa = ", k)
    print("accuracy = ", a)
    Lk.append(k)
    La.append(a)
plt.plot(r, errors, label = "log loss")
plt.legend(loc='lower right')
plt.show()
#plt.plot(r, La, label = "accuracy")
plt.plot(r, Lk, label = "kappa")
plt.legend(loc='lower right')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17072919527602307, 'fraction_numerical': 0.06512633891787971, 'mean_word_length': 3.6097483779078967, 'pattern_counts': {'":': 0, '<': 57, '<?xml version=': 0, '>': 58, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '123964', 'license': 'Apache-2.0', 'n_tokens_mistral': 13537, 'n_tokens_neox': 12270, 'n_words': 2319, 'path': 'notebooks/N epochs Amplitude .ipynb', 'repo_name': 'Emmanuel75/Dreem_Data_challenge_2018', 'size': '29128'}