<jupyter_start><jupyter_text>Read Data<jupyter_code>Train = pd.read_csv('Bidirectional_Botnet_Training_Final_Flow_Based_Features.csv',verbose=False)
Test = pd.read_csv('Bidirectional_Botnet_Test_Final_Flow_Based_Features.csv',verbose=False)
print 'Done Reading'
features = ['APL',
 'AvgPktPerSec',
 'IAT',
 'NumForward',
 'Protocol',
 'BytesEx',
 'BitsPerSec',
 'NumPackets',
 'StdDevLen',
 'SameLenPktRatio',
 'FPL',
 'Duration',
 'NPEx']

target = 'isBot'
Train = Train[features+[target]]
Test = Test[features+[target]]
Train.head(3)
ntrain = Train.shape[0]
Tr_Te = pd.concat((Train,Test),axis=0)
num_features = ['APL',
 'AvgPktPerSec',
 'IAT',
 'NumForward',
 'BytesEx',
 'BitsPerSec',
 'NumPackets',
 'StdDevLen',
 'SameLenPktRatio',
 'FPL',
 'Duration',
 'NPEx']
cat_features = ['Protocol']<jupyter_output><empty_output><jupyter_text>Dummify categorical variables and normalize numerical<jupyter_code>X = []
##Categorical Varialbes
for x in cat_features:
    temp = pd.get_dummies(Tr_Te[x].astype('category'))
    X.append(temp)

scaler = StandardScaler()
tmp = scaler.fit_transform(Tr_Te[num_features])
X.append(tmp)<jupyter_output><empty_output><jupyter_text>Extract Target labes<jupyter_code>Y = Tr_Te['isBot']<jupyter_output><empty_output><jupyter_text>Remove unnecessary variables<jupyter_code>del(Tr_Te,Train,Test)
temp = X[0]
for i in range(1,len(X)):
    temp = np.hstack((temp,X[i]))
    
import copy
X = copy.deepcopy(temp)
print X.shape
del(temp)
X_train = X[:ntrain,:]
X_test = X[ntrain:,:]
Y_train = Y[:ntrain]
Y_test = Y[ntrain:]
del(X)
X_train
Y_train
print len(X_train),len(Y_train)
print len(X_test),len(Y_test)
from keras import backend as K
def custom_obj(y_true, y_pred):
    '''Calculates the Matthews correlation coefficient measure for quality
    of binary classification problems.
    '''
    y_pred_pos = K.round(K.clip(y_pred, 0, 1))
    y_pred_neg = 1 - y_pred_pos

    y_pos = K.round(K.clip(y_true, 0, 1))
    y_neg = 1 - y_pos

    tp = K.sum(y_pos * y_pred_pos)
    tn = K.sum(y_neg * y_pred_neg)

    fp = K.sum(y_neg * y_pred_pos)
    fn = K.sum(y_pos * y_pred_neg)

    return 2.0*fn*fp/(fn+fp)

from keras import backend as K
def matthews_correlation(y_true, y_pred):
    '''Calculates the Matthews correlation coefficient measure for quality
    of binary classification problems.
    '''
    y_pred_pos = K.round(K.clip(y_pred, 0, 1))
    y_pred_neg = 1 - y_pred_pos

    y_pos = K.round(K.clip(y_true, 0, 1))
    y_neg = 1 - y_pos

    tp = K.sum(y_pos * y_pred_pos)
    tn = K.sum(y_neg * y_pred_neg)

    fp = K.sum(y_neg * y_pred_pos)
    fn = K.sum(y_pos * y_pred_neg)

    numerator = (tp * tn - fp * fn)
    denominator = K.sqrt((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn))

    return numerator / (denominator + K.epsilon())

def nn_model():
    model = Sequential()
    
    model.add(Dense(100, input_dim = X_train.shape[1], init = 'he_normal'))
    model.add(PReLU())
    model.add(BatchNormalization())
        
    model.add(Dense(50, init = 'he_normal'))
    model.add(PReLU())
    model.add(BatchNormalization())    
    
    model.add(Dense(50, init = 'he_normal'))
    model.add(Activation('sigmoid'))
    model.add(BatchNormalization())    
    
    model.add(Dense(1, init = 'he_normal'))
    model.compile(loss = 'binary_crossentropy', optimizer = 'adam',metrics=['accuracy','fbeta_score','matthews_correlation'])
    return(model)

def nn_model_dropout():
    model = Sequential()
    
    model.add(Dense(100, input_dim = X_train.shape[1], init = 'he_normal'))
    model.add(Activation('sigmoid'))
    model.add(BatchNormalization())
    model.add(Dropout(0.4))
        
    model.add(Dense(50, init = 'he_normal'))
    model.add(Activation('sigmoid'))
    model.add(BatchNormalization())    
    model.add(Dropout(0.2))
    
    model.add(Dense(20, init = 'he_normal'))
    model.add(Activation('sigmoid'))
    model.add(BatchNormalization())    
    model.add(Dropout(0.2))
    
    model.add(Dense(1, init = 'he_normal'))
    model.compile(loss = 'binary_crossentropy', optimizer = 'adam',metrics=['accuracy','fbeta_score','matthews_correlation'])
    return(model)
model = nn_model_dropout()
csv_logger = CSVLogger('log.txt')
checkpointer = ModelCheckpoint(filepath="Models/Best.hdf5", verbose=1, save_best_only=True)
earlyStopping = EarlyStopping(monitor='val_acc', patience=10, verbose=2, mode='min')
model.fit(X_train,Y_train,nb_epoch=100,batch_size=128,callbacks=[csv_logger,checkpointer],validation_data=(X_test,Y_test),verbose=1)
y_pred = model.predict_classes(X_train)
y_pred = np.reshape(y_pred,(y_pred.shape[0]))
true_pred = np.array(Y_train)
def print_metr(y_pred,y_true):
    print '\n',classification_report(y_pred,y_true)
print_metr(y_pred,true_pred)
pred_test = model.predict_classes(X_test)
true_test = np.reshape(Y_test,(Y_test.shape[0]))
print_metr(pred_test,true_test)
confusion_matrix(pred_test,true_test)
sum(Y_test)
len(Y_test)-sum(Y_test)
print sum(Y_train),len(Y_train)-sum(Y_train)<jupyter_output>185311 318733
{'fraction_non_alphanumeric': 0.15184815184815184, 'fraction_numerical': 0.013786213786213787, 'mean_word_length': 5.002398081534772, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '606609', 'license': 'MIT', 'n_tokens_mistral': 2012, 'n_tokens_neox': 1937, 'n_words': 313, 'path': 'Botnet Detection - Extraction, Analysis and ML/Deep Learning 30-11/.ipynb_checkpoints/DL-checkpoint.ipynb', 'repo_name': 'apexsodomy/Botnet-Detection-using-Machine-Learning', 'size': '5005'}