<jupyter_start><jupyter_text>Expectation Reflection for Classification<jupyter_code>import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold
from sklearn.utils import shuffle
from sklearn.metrics import accuracy_score

import expectation_reflection as ER
from sklearn.linear_model import LogisticRegression

import matplotlib.pyplot as plt
%matplotlib inline
np.random.seed(1)
def synthesize_data(l,n,g,data_type='discrete'):        
    if data_type == 'binary':
        X = np.sign(np.random.rand(l,n)-0.5)
        w = np.random.normal(0.,g/np.sqrt(n),size=n)
        
    if data_type == 'continuous':
        X = 2*np.random.rand(l,n)-1
        w = np.random.normal(0.,g/np.sqrt(n),size=n)
        
    if data_type == 'categorical':        
        from sklearn.preprocessing import OneHotEncoder
        m = 5 # categorical number for each variables
        # initial s (categorical variables)
        s = np.random.randint(0,m,size=(l,n)) # integer values
        onehot_encoder = OneHotEncoder(sparse=False,categories='auto')
        X = onehot_encoder.fit_transform(s)
        w = np.random.normal(0.,g/np.sqrt(n),size=n*m)
        
    h = X.dot(w)
    p = 1/(1+np.exp(-2*h)) # kinetic
    #p = 1/(1+np.exp(-h))  # logistic regression
    y = np.sign(p - np.random.rand(l))
    return w,X,y
from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import GaussianNB
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier

def ML_inference(X_train,y_train,X_test,y_test,method='expectation_reflection'):
    if method == 'expectation_reflection':
        h0,w = ER.fit(X_train,y_train,niter_max=100,regu=0.001)
        y_pred = ER.predict(X_test,h0,w)
        accuracy = accuracy_score(y_test,y_pred)
        
    else:
        if method == 'logistic_regression':
            model = LogisticRegression(solver='liblinear')

        if method == 'naive_bayes': 
            model = GaussianNB()

        if method == 'random_forest':
            model = RandomForestClassifier(criterion = "gini", random_state = 1,
                           max_depth=3, min_samples_leaf=5,n_estimators=100)        
        if method == 'decision_tree':
            model = DecisionTreeClassifier()  

        model.fit(X_train, y_train)
        y_pred = model.predict(X_test)        
        accuracy = accuracy_score(y_test,y_pred)
    return accuracy
list_methods=['expectation_reflection','naive_bayes','logistic_regression','decision_tree','random_forest']
def compare_ML_inference(X,y,train_size):
    npred = 10
    accuracy = np.zeros((len(list_methods),npred))
    for ipred in range(npred):
        X, y = shuffle(X, y)
        X_train0,X_test,y_train0,y_test = train_test_split(X,y,test_size=0.2,random_state = ipred)

        idx_train = np.random.choice(len(y_train0),size=int(train_size*len(y_train0)),replace=False)
        X_train,y_train = X_train0[idx_train],y_train0[idx_train]

        for i,method in enumerate(list_methods):
            accuracy[i,ipred] = ML_inference(X_train,y_train,X_test,y_test,method)
            #print('% 20s :'%method,accuracy)            
        #print(y_train.shape[0],y_test.shape[0])

    return accuracy.mean(axis=1),accuracy.std(axis=1)<jupyter_output><empty_output><jupyter_text>Categorical variables<jupyter_code>l = 10000
n = 20
g = 30.
w0,X,y = synthesize_data(l,n,g,data_type='categorical')
from sklearn.preprocessing import MinMaxScaler
X = MinMaxScaler().fit_transform(X)
list_train_size = [1.,0.8,0.6,0.4,0.2,0.1,0.05]

acc = np.zeros((len(list_train_size),len(list_methods)))
acc_std = np.zeros((len(list_train_size),len(list_methods)))
for i,train_size in enumerate(list_train_size):
    acc[i,:],acc_std[i,:] = compare_ML_inference(X,y,train_size)
    print(train_size,acc[i,:])
plt.figure(figsize=(4,3))
plt.plot(list_train_size,acc[:,0],'k-',label='ER')
plt.plot(list_train_size,acc[:,1],'b-',label='Naive Bayes')
plt.plot(list_train_size,acc[:,2],'r-',label='Logistic Regression')
plt.plot(list_train_size,acc[:,3],'b--',label='Decision Tree')
plt.plot(list_train_size,acc[:,4],'r--',label='Random Forest')
plt.xlabel('train-size')
plt.ylabel('acc')
plt.legend()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13637426900584795, 'fraction_numerical': 0.01801169590643275, 'mean_word_length': 4.001169590643275, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '66684', 'license': 'MIT', 'n_tokens_mistral': 1595, 'n_tokens_neox': 1534, 'n_words': 231, 'path': '.ipynb_checkpoints/1main_npred100_all_categorical_g30-checkpoint.ipynb', 'repo_name': 'danhtaihoang/expectation-reflection', 'size': '4275'}