<jupyter_start><jupyter_text>outlier treartment<jupyter_code>df01_continous = df01_continous.apply(lambda x: x.clip(lower = x.quantile(0.01), upper = x.quantile(0.99)))
temp= df01_continous.apply(lambda x: x.clip(upper = x.quantile(0.985)))
temp1= pd.concat([temp, df01_categorical], axis=1, join='inner')<jupyter_output><empty_output><jupyter_text>high cardinality treatment service<jupyter_code>from sklearn import preprocessing
le = preprocessing.LabelEncoder()
le.fit(temp1['service'])<jupyter_output><empty_output><jupyter_text>dummy variable creation<jupyter_code>categorial_features= ['protocol_type', 'flag']
# An utility function to create dummy variable
def create_dummies( df, colname ):
    col_dummies = pd.get_dummies(df[colname], prefix=colname)
    col_dummies.drop(col_dummies.columns[0], axis=1, inplace=True)
    df = pd.concat([df, col_dummies], axis=1)
    df.drop( colname, axis = 1, inplace = True )
    return df
for c_feature in categorial_features:
    temp1= create_dummies(temp1, c_feature)
temp_test_2= temp1.copy()
temp_test_2.attack= np.where(temp_test_2.attack== 'normal', 1, 0)
#temp_test_2.drop_duplicates(keep='first', inplace= True)
temp_test_2.service= le.transform(temp1['service']) 
temp_test_2.info()
temp_test_2.to_excel('Normal Classification Df.xlsx', index=False)<jupyter_output><empty_output><jupyter_text>Model building<jupyter_code>from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
feature_columns= temp_test_2.columns.difference(["hot","num_failed_logins","logged_in",
"num_compromised","root_shell","su_attempted","num_root","num_file_creations",
"num_shells","num_access_files","num_outbound_cmds","is_host_login",
"is_guest_login","count","srv_count","serror_rate", "srv_serror_rate",
"rerror_rate","srv_rerror_rate","same_srv_rate", "diff_srv_rate",
"srv_diff_host_rate","dst_host_count","dst_host_srv_count","dst_host_same_srv_rate",
"dst_host_diff_srv_rate","dst_host_same_src_port_rate",
"dst_host_srv_diff_host_rate","dst_host_serror_rate","dst_host_srv_serror_rate",
"dst_host_rerror_rate","dst_host_srv_rerror_rate","attack", "last_flag" ])
train_X, test_X, train_y, test_y = train_test_split( temp_test_2[feature_columns],
                                                  temp_test_2["attack"],
                                                  test_size = 0.3,
                                                  random_state = 123 )
train_X.head()

logreg = LogisticRegression()
logreg.fit( train_X, train_y )

logreg.predict(train_X)   #by default, it use cut-off as 0.5
list( zip( feature_columns, logreg.coef_[0] ) )
logreg.intercept_
#Predicting the test cases
network_test_pred = pd.DataFrame( { 'actual':  test_y,
                            'predicted': logreg.predict( test_X ) } )
network_test_pred= network_test_pred.reset_index()
network_test_pred.to_excel('network_test_predictions using logistic.xlsx', index=False)
network_test_pred.sample(n=10)
# Creating a confusion matrix

from sklearn import metrics

cm = metrics.confusion_matrix( network_test_pred.actual,
                            network_test_pred.predicted, [1,0] )
cm
import seaborn as sn
import matplotlib.pyplot as plt
sn.heatmap(cm, annot=True,  fmt='.2f', xticklabels = ["Left", "No Left"] , yticklabels = ["Left", "No Left"] )
plt.ylabel('True label')
plt.xlabel('Predicted label')
score = metrics.accuracy_score( network_test_pred.actual, network_test_pred.predicted )
round( float(score), 2 )
# Predit Probability
print(test_X[:1])
print(logreg.predict_proba( test_X[:1] ))
#How good the model is?
predict_proba_df = pd.DataFrame( logreg.predict_proba( test_X ) )
predict_proba_df.head()
network_test_pred = pd.concat( [network_test_pred, predict_proba_df], axis = 1 )
network_test_pred.columns = ['index', 'actual', 'predicted', 'Attack_0', 'Attack_1']

network_test_pred.to_excel('Final logistic normal output.xlsx')
auc_score = metrics.roc_auc_score( network_test_pred.actual, network_test_pred.Attack_1 )
float( auc_score )
sn.distplot( network_test_pred[network_test_pred.actual == 1]["Attack_1"], color = 'b' )
sn.distplot( network_test_pred[network_test_pred.actual == 0]["Attack_1"], color = 'g' )
# Finding the optimal cutoff probability
fpr, tpr, thresholds = metrics.roc_curve( network_test_pred.actual,
                                     network_test_pred.Attack_1,
                                     drop_intermediate = False )

plt.figure(figsize=(6, 4))
plt.plot( fpr, tpr, label='ROC curve (area = %f)' % auc_score )
plt.plot([0, 1], [0, 1], 'k--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate or [1 - True Negative Rate]')
plt.ylabel('True Positive Rate')
plt.title('Receiver operating characteristic example')
plt.legend(loc="lower right")
plt.show()
print(thresholds[0:10])
print(fpr[0:10])
print(tpr[0:10])
cutoff_prob = thresholds[(np.abs(tpr - 0.9)).argmin()]
float( cutoff_prob )
round( float( cutoff_prob ), 2 )
pandas_profiling.ProfileReport(temp1).to_file(output_file='normalvsrest.html')<jupyter_output><empty_output><jupyter_text>Multi Class Classification<jupyter_code>temp1.attack.loc[temp1.attack== 'normal']= 'Normal'
temp1.attack.loc[temp1.attack.isin(['back','land', 'neptune', 'teardrop', 'smurf', 'pod', 'apache2', 'udpstorm', 'processtable', 'worm'])] ='DoS'
temp1.attack.loc[temp1.attack.isin(['satan', 'ipsweep', 'nmap', 'portsweep', 'mscan', 'saint'])]= 'Probe'
temp1.attack.loc[temp1.attack.isin(['guess_passwd', 'ftp_write', 'imap', 'phf', 'multihop', 'warezmaster', 'warezclient', 'spy', 'xlock', 'xsnoop'
                  'snmguess', 'snmpgetattack', 'httptunnel', 'sendmail', 'named'])]= 'R2L'
temp1.attack.loc[temp1.attack.isin(['buffer_overflow', 'loadmodule', 'rootkit', 'perl', 'sqlattack', 'xterm', 'ps'])]= 'U2R'
temp1.attack.unique()
temp1.service= le.transform(temp1['service']) 
temp1.drop_duplicates(keep='first', inplace=True)
onevrest= temp1.copy()
from sklearn.metrics import roc_curve, auc
from sklearn.multiclass import OneVsRestClassifier
from sklearn.svm import LinearSVC
from sklearn.preprocessing import label_binarize
from sklearn.model_selection import GridSearchCV
onevsrest_y= label_binarize(onevrest.attack, classes=['Normal', 'DoS', 'R2L', 'Probe', 'U2R']) 
n_classes= 5
X_train_ovr, X_test_ovr, y_train_ovr, y_test_ovr =\
    train_test_split(onevrest[feature_columns], onevsrest_y, test_size=0.33, random_state=0)
# classifier model
clf = OneVsRestClassifier(LinearSVC(random_state=0))
y_score = clf.fit(X_train_ovr, y_train_ovr).decision_function(X_test_ovr)


# Compute ROC curve and ROC area for each class
fpr = {}
tpr = {}
roc_auc = {}
for i in range(n_classes):
    fpr[i], tpr[i], _ = roc_curve(y_test_ovr[:, i], y_score[:, i])
    roc_auc[i] = auc(fpr[i], tpr[i])
# # Plot of a ROC curve for a specific class 
# for i in range(n_classes):
#     plt.figure()
#     plt.plot(fpr[i], tpr[i], label='ROC curve (area = %0.2f)' % roc_auc[i])
#     plt.plot([0, 1], [0, 1], 'k--')
#     plt.xlim([0.0, 1.0])
#     plt.ylim([0.0, 1.05])
#     plt.xlabel('False Positive Rate')
#     plt.ylabel('True Positive Rate')
#     plt.title('Receiver operating characteristic example')
#     plt.legend(loc="lower right")
#     plt.show()
# Plot of a ROC curve for a specific class
for i in range(n_classes):
    plt.figure()
    plt.plot(fpr[i], tpr[i], label='ROC curve (area = %0.2f)' % roc_auc[i])
    plt.plot([0, 1], [0, 1], 'k--')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('Receiver operating characteristic example')
    plt.legend(loc="lower right")
    plt.show()
#model = clf.fit(X_train_ovr, y_train_ovr)
#y_score = clf.fit(X_train_ovr, y_train_ovr).decision_function(X_test_ovr)
classifir= OneVsRestClassifier(LinearSVC()).fit(X_train_ovr, y_train_ovr)

output= classifir.predict(X_test_ovr)
df2=pd.DataFrame(np.array(output),
                   columns=['class 0 Predicted', 'Class 1 Predicted', 'Class 2 Predicted', 'Class 3 Predicted', 'Class4 Predicted'])
y_test_ovr
df3= pd.DataFrame(y_test_ovr, columns=['class 0 Actual', 'Class 1 Actual', 'Class 2 Actual', 'Class 3 Actual', 'Class4 Actual'])
out= pd.concat([df2, df3], axis=1)
out.to_excel('final output logistic multiclass.xlsx')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14941148210425173, 'fraction_numerical': 0.01993754503963488, 'mean_word_length': 5.425154320987654, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '226873', 'license': 'MIT', 'n_tokens_mistral': 3218, 'n_tokens_neox': 3071, 'n_words': 566, 'path': 'normal classification.ipynb', 'repo_name': 'HardeepSiinghh/Network-Intrusion-Detection-System', 'size': '8326'}