<jupyter_start><jupyter_text>Processing for skewed dataFeature Selection via Shapley ValuesTraining Decision Tree ClassifierTraining Vanilla NNTraining Multiple experts systems  Processing for data after outlier removal and ImputationFeature Selection via Shapley ValuesTraining Decision Tree ClassifierTraining Vanilla NNTraining Multiple experts systems<jupyter_code>from google.colab import drive
drive.mount('/content/drive')
%cd /content/drive/MyDrive
!pip install shap
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.preprocessing import MinMaxScaler
from mpl_toolkits.mplot3d import Axes3D
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
import collections
from sklearn.naive_bayes import GaussianNB
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.model_selection import  cross_val_score
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import classification_report
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from xgboost.sklearn import XGBRegressor
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.linear_model import LinearRegression
from sklearn.tree import DecisionTreeRegressor
from sklearn import tree
from sklearn.ensemble import RandomForestRegressor
from xgboost import XGBClassifier
from sklearn.tree import DecisionTreeClassifier
import shap
from sklearn.metrics import mean_squared_error

%matplotlib inline
# import os
# os.listdir()
df=pd.read_csv('complete_data.csv')
df.head()
#SHAPLEY
y=df['output_nonad']
X=df.drop('output_nonad', 1)

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.25, random_state=42)

shap.initjs()
xgb_model = XGBRegressor(n_estimators=1000, max_depth=10, learning_rate=0.001, random_state=0)
xgb_model.fit(X_train, y_train)

y_predict = xgb_model.predict(X_test)
mean_squared_error(y_test, y_predict)**(0.5)

explainer = shap.TreeExplainer(xgb_model)
shap_values = explainer(X_train)
print(shap_values)


shap.plots.bar(shap_values,max_display=16)
cols=['width','ancurl*com','url*ads','height','ancurl*click','aratio','url*ad','ancurl*http+www','alt*click','alt*here+for','ancurl*ad','url*afn.org','alt*to','url*images+home','url*doubleclick.net','output_nonad']
cols_X=['width','ancurl*com','url*ads','height','ancurl*click','aratio','url*ad','ancurl*http+www','alt*click','alt*here+for','ancurl*ad','url*afn.org','alt*to','url*images+home','url*doubleclick.net']
datapd=df[['width','url*ads','height','ancurl*click','aratio','url*ad','ancurl*http+www','alt*here+for','ancurl*ad','url*afn.org','alt*to','url*images+home','output_nonad']]
# fig,ax=plt.subplots(nrows=1,ncols=3)
# fig.set_figheight(5)
# fig.set_figwidth(25)



# sns.distplot(datapd['aratio'],ax=ax[0])
# sns.distplot(datapd['height'],ax=ax[1])
# sns.distplot(datapd['width'],ax=ax[2])

columns_for_visualization = list()
columns_for_visualization=cols_X
corr=datapd[columns_for_visualization].corr()
fig, ax = plt.subplots(figsize=(15,15))
sns.heatmap(corr,annot=True,linewidths=.5, ax=ax)
da=datapd.values
X=da[:,:-1]
y=da[:,-1]


xtrain,xtest,ytrain,ytest=train_test_split(X,y,test_size=0.30,random_state=80)

def fit_models(classifiers,xtrain,ytrain):
    """This function fit multiple models by sklearn and return the dictionary with values as  objects of models"""
    models=collections.OrderedDict()
    for constructor in classifiers:
        obj=constructor()
        obj.fit(xtrain,ytrain)
        models[str(constructor).split(':')[0]]=obj
    return models

def classification_multi_report(ytest,models_array):
    """This function generate classification accuracy report for given input model objects"""
    for i in models_array:
        print('__________________________________________________')
        print('the model - '+str(i))
        print(classification_report(ytest,models_array[i].predict(xtest)))
def cross_Fucntion(models,cv):
    """This function return cross validated accuray and the variance of given input model obejects"""
    accuracy={}
    for model in models:
        cross_val_array=cross_val_score(models[model],xtrain,ytrain,scoring='accuracy',cv=cv)
        accuracy[model]=[np.mean(cross_val_array),np.std(cross_val_array)]
    return accuracy

def multi_grid_search(param_grid_array,estimator_list,x,y):
    """This function calculate the grid search parameters and accuracy  for given input modles and return dictionary with each tupple containing accuracy and best parameters"""
    d={}
    count=0
    for i in estimator_list:
        gc=GridSearchCV(estimator=estimator_list[i],param_grid=param_grid_array[count],scoring ='accuracy',cv=5).fit(x,y)
        d[i]=(gc.best_params_,gc.best_score_)
        count+=1
    return d<jupyter_output><empty_output><jupyter_text>Classification  Report>The classification report visualizer displays the precision, recall, F1, and support scores for the model. ... Visual classification reports are used to compare classification models to select models that are “redder”, e.g. have stronger classification metrics or that are more balanced.<jupyter_code>classifiers=[ SVC,KNeighborsClassifier, RandomForestClassifier, GradientBoostingClassifier,LogisticRegression,XGBClassifier,DecisionTreeClassifier]
# classifiers=[DecisionTreeClassifier]

model_list=fit_models(classifiers,xtrain,ytrain)

classification_multi_report(ytest,model_list)
obj=cross_Fucntion(model_list,cv=20)
for model in obj:
    print('the model -'+str(model)+'has \n || crosss validated accuracy as  -> '+str(obj[model][0])+' | variance - '+str(obj[model][1])+' ||' )
    print('______________________________________________________________________________________________________________')

param_grid_svm=[
    {
        'kernel':['linear'],'random_state':[0]
    },
     {
        'kernel':['rbf'],'random_state':[0]
     },
    
    {
        'kernel':['poly'],'degree':[1,2,3,4],'random_state':[0]
    }
]

param_grid_knn=[

    {   
        'n_neighbors':[1,5,11,15,20,25],
        'p':[2]
        
    }
]

param_grid_nb=[
    {}
]
#RandomF SVC,KNeighborsClassifier, RandomForestClassifier, GradientBoostingClassifier,LogisticRegressio,XGB

param_grid_rf=[
               {
                   'n_estimators': np.arange(100, 1000, 100),
                  # 'max_depth' : np.arange(5, 100,5)
               }
]

param_grid_xgb=[
               {
                   'n_estimators': np.arange(100,500, 100)
                  
               }
]

param_grid_lr=[
              {}
]
param_grid_xgbx=[
               {
                   'n_estimators': np.arange(100,500, 100)
                  
               }
]

param_grid_array=[param_grid_svm,param_grid_knn,param_grid_rf,param_grid_xgb,param_grid_lr,param_grid_xgbx]
multi_grid_search(param_grid_array,model_list,xtrain,ytrain)<jupyter_output><empty_output><jupyter_text>Fitting model with best hyperparmater and cv score<jupyter_code>classifier=RandomForestClassifier(n_estimators=800)
classifier.fit(xtrain,ytrain)

sns.heatmap(pd.crosstab(ytest,classifier.predict(xtest)),cmap='coolwarm')
plt.xlabel('predicted')
plt.ylabel('actual')
plt.show()
#custom NN
from keras.models import Sequential
from keras.layers import Dense, Dropout
from keras.optimizers import RMSprop,Adam

print(X.shape)
X_train1, X_test, y_train1, y_test = train_test_split(X, y, test_size=0.2, random_state=1)

X_train, X_val, y_train, y_val = train_test_split(X_train1, y_train1, test_size=0.125, random_state=1)
model=Sequential() 
model.add(Dense(25,activation='relu',input_shape=(15,),kernel_initializer='he_normal'))
model.add(Dense(20,activation='sigmoid'))
# model.add(Dense(20,activation='sigmoid'))
# model.add(Dropout(0.2))
# model.add(Dense(20,activation='sigmoid'))
# model.add(Dropout(0.2))
# model.add(Dense(15,activation='sigmoid'))
model.add(Dense(5,activation='sigmoid'))
# model.add(Dropout(0.2))
# model.add(Dropout(0.2))

# model.add(Dropout(0.2))
# model.add(Dense(90,activation='relu'))
# model.add(Dropout(0.2))
# model.add(Dense(60,activation='relu'))
# model.add(Dropout(0.2))
# model.add(Dense(30,activation='relu'))
# model.add(Dropout(0.2))
model.add(Dense(1,activation='sigmoid'))
# model.add(DropOut(0.2))
model.summary()
print(X_train.shape)

model.compile(loss='binary_crossentropy',
              optimizer=Adam(),
              metrics=['accuracy','AUC'])
class_w={0.0:8,1.0:1}

history = model.fit(X_train1, y_train1,class_weight=class_w,
                    batch_size=32,
                    epochs=100,
                    verbose=1,
                    validation_data=(X_val, y_val))

score = model.evaluate(X_test,y_test, verbose=0)
print(score)
print('Test accuracy:', score[1])
model.save('model_weights_Skewed_2.h5')
%cd /content/drive/MyDrive
import numpy as np
import tensorflow as tf
import tensorflow.keras.backend as K
from tensorflow import keras
from tensorflow.keras import layers
import pandas as pd
from sklearn.model_selection import train_test_split
synth_df=pd.read_csv('complete_data.csv')

X=synth_df.values[:,:-1]
y=synth_df.values[:,-1]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.20, random_state=42)
geo_X_train=X_train[:,0:4]
geo_X_test=X_test[:,0:4]
url_X_train=X_train[:,4:461]
url_X_test=X_test[:,4:461]
origurl_X_train=X_train[:,461:956]
origurl_X_test=X_test[:,461:956]
ancurl_X_train=X_train[:,956:1428]
ancurl_X_test=X_test[:,956:1428]
alt_X_train=X_train[:,1428:1539]
alt_X_test=X_test[:,1428:1539]
cap_X_train=X_train[:,1539:1558]
cap_X_test=X_test[:,1539:1558]
geo_input=keras.Input(shape=(4,),name='geo')
# geo_x=layers.Dense(16,activation='sigmoid')(geo_input)
geo_x=layers.Dense(8,activation='sigmoid')(geo_input)
geo_output=layers.Dense(1,activation='sigmoid')(geo_x)
url_input=keras.Input(shape=(457,),name='url') 
# url_x=layers.Dense(256,activation='sigmoid')(url_input)
url_x=layers.Dense(20,activation='sigmoid')(url_input)
url_output=layers.Dense(1,activation='sigmoid')(url_x)
origurl_input=keras.Input(shape=(495,),name='origurl') 
# origurl_x=layers.Dense(256,activation='sigmoid')(origurl_input)
origurl_x=layers.Dense(20,activation='sigmoid')(origurl_input)
origurl_output=layers.Dense(1,activation='sigmoid')(origurl_x)
ancurl_input=keras.Input(shape=(472,),name='ancurl') 
# ancurl_x=layers.Dense(256,activation='sigmoid')(ancurl_input)
ancurl_x=layers.Dense(20,activation='sigmoid')(ancurl_input)
ancurl_output=layers.Dense(1,activation='sigmoid')(ancurl_x)
alt_input=keras.Input(shape=(111,),name='alt') 
# alt_x=layers.Dense(64,activation='sigmoid')(alt_input)
alt_x=layers.Dense(20,activation='sigmoid')(alt_input)
alt_output=layers.Dense(1,activation='sigmoid')(alt_x)
cap_input=keras.Input(shape=(19,),name='cap') 
# cap_x=layers.Dense(16,activation='sigmoid')(cap_input)
cap_x=layers.Dense(8,activation='sigmoid')(cap_input)
cap_output=layers.Dense(1,activation='sigmoid')(cap_x)
xf1=layers.concatenate([geo_output,url_output,origurl_output,ancurl_output,alt_output,cap_output])
# xf1=layers.Dense(4,activation='sigmoid')(xf1)
result=layers.Dense(1, name="ad/nonad")(xf1)
model=keras.Model(inputs=[geo_input,url_input,origurl_input,ancurl_input,alt_input,cap_input],outputs=[result])
keras.utils.plot_model(model, "multiModal_skewed_3.png", show_shapes=True)

model.summary()
model.compile(
    optimizer=keras.optimizers.RMSprop(1e-3),
    loss=[
        keras.losses.BinaryCrossentropy(from_logits=True),
        # keras.losses.CategoricalCrossentropy(from_logits=True),
    ],
    metrics=['accuracy'],
    
)
s='geo_X_test'
print(s.replace("geo","geo"),",")
print(s.replace("geo","url"),",",)
print(s.replace("geo","origurl"),",",)
print(s.replace("geo","ancurl"),",",)
print(s.replace("geo","alt"),",",)
print(s.replace("geo","cap"),"",)
model.fit(
    {"geo": geo_X_train, "url": url_X_train ,
      "origurl": origurl_X_train ,
      "ancurl": ancurl_X_train ,
      "alt": alt_X_train ,
      "cap": cap_X_train },
    {"ad/nonad": y_train},
    epochs=200,
    batch_size=64,
    validation_split=0.125,
    verbose=1,

)
test_score= model.predict([geo_X_test ,
url_X_test ,
origurl_X_test ,
ancurl_X_test ,
alt_X_test ,
cap_X_test ])
def custom_f1(y_true, y_pred):
  tn=0
  tp=0
  fn=0
  fp=0    
  for i in range(656):
    # print(y_true[i][0],)
    if y_true[i][0]==0 and y_pred[i][0]==0:
      tn+=1
    if y_true[i][0]==0 and y_pred[i][0]==1:
      fp+=1
    if y_true[i][0]==1 and y_pred[i][0]==0:
      fn+=1
    if y_true[i][0]==1 and y_pred[i][0]==1:
      tp+=1
  print(tn,tp,fn,fp)
  precision=tp/(tp+fp)
  recall=tp/(tp+tn)
  print(recall)
  print(precision)
  print((tp+tn)/(tp+tn+fp+fn))
  return 2*((precision*recall)/(precision+recall))

print(custom_f1(y_test,test_score))

for i in range(len(test_score)):
  if test_score[i]>=0:
    test_score[i]=1
  else:
    test_score[i]=0
model.save('multimodal_skewed_6grps.h5')
y_test=np.reshape(y_test,(656,1))
%cd /content/drive/MyDrive
import numpy as np
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
import pandas as pd
from sklearn.model_selection import train_test_split
synth_df=pd.read_csv('complete_data.csv')
X=synth_df.values[:,:-1]
y=synth_df.values[:,-1]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.20, random_state=42)
geo_X_train=X_train[:,0:4]
geo_X_test=X_test[:,0:4]
url_X_train=X_train[:,4:1428]
url_X_test=X_test[:,4:1428]
alt_X_train=X_train[:,1428:1558]
alt_X_test=X_test[:,1428:1558]
geo_input=keras.Input(shape=(4,),name='geo')
# geo_x=layers.Dense(16,activation='sigmoid')(geo_input)
geo_x=layers.Dense(8,activation='sigmoid')(geo_input)
geo_output=layers.Dense(1,activation='sigmoid')(geo_x)
url_input=keras.Input(shape=(1424,),name='url') 
url_x=layers.Dense(320,activation='sigmoid')(url_input)
url_x=layers.Dense(64,activation='sigmoid')(url_x)
url_output=layers.Dense(1,activation='sigmoid')(url_x)
alt_input=keras.Input(shape=(130,),name='alt') 
alt_x=layers.Dense(64,activation='sigmoid')(alt_input)
alt_x=layers.Dense(20,activation='sigmoid')(alt_x)
alt_output=layers.Dense(1,activation='sigmoid')(alt_x)
xf1=layers.concatenate([geo_output,url_output,alt_output])
xf1=layers.Dense(2,activation='sigmoid')(xf1)
result=layers.Dense(1, name="ad/nonad")(xf1)
model=keras.Model(inputs=[geo_input,url_input,alt_input],outputs=[result])
keras.utils.plot_model(model, "multi_modal_skewed_2.png", show_shapes=True)
model.compile(
    optimizer=keras.optimizers.RMSprop(1e-3),
    loss=[
        keras.losses.BinaryCrossentropy(from_logits=True),
        # keras.losses.CategoricalCrossentropy(from_logits=True),
    ],
    metrics=['accuracy'],
    
)
model.fit(
    {"geo": geo_X_train, "url": url_X_train ,
      "alt": alt_X_train ,
       },
    {"ad/nonad": y_train},
    epochs=400,
    batch_size=64,
    validation_split=0.125,
    verbose=2,

)
test_score= model.evaluate([geo_X_test ,
url_X_test ,
alt_X_test  ],y_test,verbose=2)
y_pred= model.predict([geo_X_test ,
url_X_test ,
alt_X_test  ])

for i in range(len(y_pred)):
  if y_pred[i][0]>=0:
    y_pred[i]=1
  else:
    y_pred[i]=0
def custom_f1(y_true, y_pred):
  tn=0
  tp=0
  fn=0
  fp=0    
  for i in range(656):
    print(y_pred[i],)
    if y_true[i]==0 and y_pred[i]==0:
      tn+=1
    if y_true[i]==0 and y_pred[i]==1:
      fp+=1
    if y_true[i]==1 and y_pred[i]==0:
      fn+=1
    if y_true[i]==1 and y_pred[i]==1:
      tp+=1
  print(tn,tp,fn,fp)
  precision=tp/(tp+fp)
  recall=tp/(tp+tn)
  print(recall)
  print(precision)
  print((tp+tn)/(tp+tn+fp+fn))
  return 2*((precision*recall)/(precision+recall))

print(custom_f1(y_test,y_pred))
model.save('skewed_multi_2.h5')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15195019692542244, 'fraction_numerical': 0.02928471604624571, 'mean_word_length': 6.33939393939394, 'pattern_counts': {'":': 11, '<': 13, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '526023', 'license': 'MIT', 'n_tokens_mistral': 6271, 'n_tokens_neox': 5968, 'n_words': 821, 'path': 'Code/Skewed_data.ipynb', 'repo_name': 'sebstian0627/Internet-Advertisement-Detection', 'size': '15742'}