<jupyter_start><jupyter_text>Model Test<jupyter_code>class logisticregression():
    def __init__(self,train_data,train_labels,lamda=0.2,lr=0.01,decay=10,batch_size=None,epoch=10,print_every = 10):
        dummy_once = np.ones((len(train_data),1))
        self.train_data = np.hstack((dummy_once,train_data))
        self.train_labels = train_labels
        
        self.params = np.zeros((len(self.train_data[0]),1))
        
        self.lr = lr
        self.epoch = epoch
        self.batch_size = batch_size
        self.print_every = print_every
        self._lambda = lamda
        self.decay = decay
        
    def sigmoid(self,x):
        return 1/(1+np.exp(-x))
    
    def cost(self,y,y_pred):
        return -np.mean(y*np.log(y_pred)+(1-y)*np.log(1-y_pred))
    
    def gradient(self,y,y_pred,x):
        return np.dot(x.T,(y_pred-y))+(2*self._lambda*self.params)
    
    def train(self):
        for i in range(self.epoch):
            y_pred = self.sigmoid(np.dot(self.train_data,self.params))
            loss = self.cost(self.train_labels,y_pred)
            
            gra = self.gradient(self.train_labels,y_pred,self.train_data)
            
            self.params -= self.lr*gra
            
            self.lr *= (1. / (1. + self.decay * i))
            
            if self.print_every:
                if i%self.print_every == 0 or i == self.epoch-1:
                    print('Epoch : {}  Loss: {}'.format(i,loss))
    def predict(self,test_data):
        result = self.sigmoid(np.dot(test_data,self.params[1:])+self.params[0])
        result[result > 0.5 ] = 1
        result[result <= 0.5 ] = 0
        return result
    
    def evaluate(self,test_data,labels):
        accuracy = accuracy_score(self.predict(test_data),labels)
        return accuracy
def cross_validate(x_data,y_data,lr,lamda=0.2,epoch=10,k=5,decay=10):
    if len(x_data)%k != 0:
        print('cant vsplit',len(x_data),' by ',k)
        return
    
    x_data_splitted = np.vsplit(x_data,k)
    y_data_splitted = np.vsplit(y_data,k)
    
    aggrigate_result = []
    for i in range(len(x_data_splitted)):
        train = []
        test = []
        items = [j for j in range(len(x_data_splitted)) if j !=i ]
        x_test = x_data_splitted[i]
        y_test = y_data_splitted[i]
        for item in items:
            if len(train) == 0:
                x_train = x_data_splitted[item]
                y_train = y_data_splitted[item]
            else:
                x_train = np.concatenate((x_train,x_data_splitted[item]), axis=0)
                y_train = np.concatenate((y_train,y_data_splitted[item]), axis=0)
            
        logistic = logisticregression(x_train,y_train,lamda=lamda,lr=lr,decay=decay,epoch=epoch)
        logistic.train()
        
        result = logistic.evaluate(x_test,y_test)
        aggrigate_result.append(result)
        
        value = sum(aggrigate_result)/len(aggrigate_result)
    return value if value!= None else 0
def objective(trial):
    lr = trial.suggest_loguniform('lr', 1e-5, 1e-1)
    lamda = trial.suggest_loguniform('lamda', 0.01, 0.5)
    k =  trial.suggest_categorical('k', [4,5,8,10])
    epoch =  trial.suggest_int('epoch', 10, 20)
    decay = trial.suggest_int('decay', 3, 10)
    return cross_validate(X_tr, y,lr=lr,lamda=lamda,k=k,epoch=epoch,decay=decay)
# cross_validate(X_train_mat100, y,0.001,10)

import optuna

sampler = optuna.samplers.TPESampler()
study = optuna.create_study(sampler=sampler, direction='maximize')
study.optimize(func=objective, n_trials=200,show_progress_bar=True)
df = study.trials_dataframe().drop(['state','datetime_start','datetime_complete'], axis=1)

df.sort_values(by=['value'])
cross_validate(X_tr,y,lamda=0.050740,epoch=10,lr=0.050740,k=4,decay=7)
# X_train, X_test, y_train, y_test
# logistic = logisticregression(X_train,y_train,lamda=0.362124,epoch=15,print_every=1,lr=0.000254)
#
logistic = logisticregression(X_train,y_train,lamda=0.05,epoch=10,lr=0.001383,decay=7,print_every=1)
logistic.train()
        
print(logistic.evaluate(X_train,y_train))
print(logistic.evaluate(X_val,y_val))

X_test_mat100 = scale(X_te)

sumbission = []
for i in range(len(X_test_mat100)):
    result = logistic.predict(X_test_mat100[i])
    sumbission.append([i,int(result)])
    result
# sumbission
df = pd.DataFrame(sumbission)
df.columns = ['Id','Bound']
df.to_csv('test_64_cross_validated.csv',index=False)
df.head(50)<jupyter_output><empty_output><jupyter_text>SVM Test<jupyter_code>from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.svm import SVC

# ‘linear’, ‘poly’, ‘rbf’, ‘sigmoid’, ‘precomputed’
clf = make_pipeline(StandardScaler(), SVC(kernel='poly'))
clf.fit(X_train,y_train)
clf.score(X_val,y_val)
from sklearn.linear_model import LogisticRegression

clf = LogisticRegression(penalty = 'l2')
clf.fit(X_train,y_train)
clf.score(X_val,y_val)<jupyter_output>/usr/local/lib/python3.6/dist-packages/sklearn/utils/validation.py:760: DataConversionWarning:

A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
{'fraction_non_alphanumeric': 0.1363547947870064, 'fraction_numerical': 0.02839914413538222, 'mean_word_length': 3.3355817875210794, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '450664', 'license': 'MIT', 'n_tokens_mistral': 1993, 'n_tokens_neox': 1907, 'n_words': 277, 'path': 'israel/solvers.ipynb', 'repo_name': 'IsraelAbebe/kernel-ammi-kaggle-Team-IS', 'size': '5141'}