<jupyter_start><jupyter_text>Prepare the data<jupyter_code>def prepare(df):
    
    y = df.label.tolist()
    X = np.matrix(df.drop(labels = ['label'], axis = 1)).astype(np.float)

    return X, y<jupyter_output><empty_output><jupyter_text>XGB<jupyter_code>from xgboost import XGBClassifier
import xgboost as xgb
trainPath = ['/scratch/by8jj/stratified samples/test of test/train/train_11.csv',
             '/scratch/by8jj/stratified samples/test of test/train/train_12.csv',
             '/scratch/by8jj/stratified samples/test of test/train/train_13.csv',
             '/scratch/by8jj/stratified samples/test of test/train/train_14.csv',
             '/scratch/by8jj/stratified samples/test of test/train/train_15.csv',
             '/scratch/by8jj/stratified samples/test of test/train/train_16.csv',
             '/scratch/by8jj/stratified samples/test of test/train/train_17.csv',
             '/scratch/by8jj/stratified samples/test of test/train/train_18.csv',
             '/scratch/by8jj/stratified samples/test of test/train/train_19.csv',
             '/scratch/by8jj/stratified samples/test of test/train/train_20.csv',
             '/scratch/by8jj/stratified samples/test of test/train/train_21.csv',
            ]
testPath = ['/scratch/by8jj/stratified samples/test of test/test/test_12.csv',
           '/scratch/by8jj/stratified samples/test of test/test/test_13.csv',
           '/scratch/by8jj/stratified samples/test of test/test/test_14.csv',
           '/scratch/by8jj/stratified samples/test of test/test/test_15.csv',
           '/scratch/by8jj/stratified samples/test of test/test/test_16.csv',
            '/scratch/by8jj/stratified samples/test of test/test/test_17.csv',
           '/scratch/by8jj/stratified samples/test of test/test/test_18.csv',
           '/scratch/by8jj/stratified samples/test of test/test/test_19.csv',
           '/scratch/by8jj/stratified samples/test of test/test/test_20.csv',
            '/scratch/by8jj/stratified samples/test of test/test/test_21.csv',
           '/scratch/by8jj/stratified samples/test of test/test/test_22.csv',
            ]
for train, test in zip(trainPath, testPath):
        
    #drop different port numbers
    df_train = pd.read_csv(train)
    temp1 = set(df_train.columns)
    
    #for test in testPath:
    df_test = pd.read_csv(test)
    temp2 = set(df_test.columns)
    
    df_train = df_train.drop(list(temp1 - temp2), axis = 1)
    df_test = df_test.drop(list(temp2 - temp1), axis = 1)
        
   
    X, y = prepare(df_train)
    
    model = XGBClassifier(silent=False, 
                      scale_pos_weight=1,
                      learning_rate=0.01,  
                      colsample_bytree = 0.4,
                      subsample = 0.8,
                      objective='binary:logistic', 
                      n_estimators=150, 
                      reg_alpha = 0.3,
                      max_depth=5,
                      verbosity = 0,
                      n_jobs = 8)
    
    X_A, X_B, Y_A, Y_B  = train_test_split(X, y, test_size=0.33)
    
    eval_set = [(X_A, Y_A), (X_B, Y_B)]
    eval_metric = ["auc","error"]
    model.fit(X_A, Y_A, eval_metric=eval_metric, eval_set=eval_set)
   
    
    X_test, y_test = prepare(df_test)
    y_pred = model.predict_proba(X_test)
    
    
    temp = [1 if x[1]>0.8 else 0 for x in y_pred]
    cm= confusion_matrix(y_test, temp)
    tn, fp, fn, tp = cm.ravel()
    precision=tp/(tp+fp)
    recall=tp/(tp+fn)
    fpr = fp/(fp+ tn)
    accuracy = (tp + tn)/(tn + tp + fn + fp)
    F1 = 2 * (precision * recall) / (precision + recall)
    print("precision:", precision*100)
    print("recall:", recall*100)
    print("false positive rate:", fpr*100)
    print("accuracy",  accuracy*100)
    print("F1-score", F1)
    
    
    pd.DataFrame(y_pred).to_csv('/scratch/by8jj/stratified samples/test of test/xgb/'+ train[-12:-4] + '-' + test[-11:-4] + '.csv', index = False)<jupyter_output>[23:20:06] /workspace/src/tree/updater_prune.cc:74: tree pruning end, 1 roots, 36 extra nodes, 0 pruned nodes, max_depth=5
[0]	validation_0-auc:0.925686	validation_0-error:0.096602	validation_1-auc:0.925962	validation_1-error:0.096693
[23:20:06] /workspace/src/tree/updater_prune.cc:74: tree pruning end, 1 roots, 40 extra nodes, 0 pruned nodes, max_depth=5
[1]	validation_0-auc:0.934292	validation_0-error:0.081063	validation_1-auc:0.934535	validation_1-error:0.081167
[23:20:07] /workspace/src/tree/updater_prune.cc:74: tree pruning end, 1 roots, 38 extra nodes, 0 pruned nodes, max_depth=5
[2]	validation_0-auc:0.938634	validation_0-error:0.095006	validation_1-auc:0.938865	validation_1-error:0.095074
[23:20:07] /workspace/src/tree/updater_prune.cc:74: tree pruning end, 1 roots, 42 extra nodes, 0 pruned nodes, max_depth=5
[3]	validation_0-auc:0.9478	validation_0-error:0.095098	validation_1-auc:0.948053	validation_1-error:0.095173
[23:20:08] /workspace/src/tree/updater_prune.cc:74: tree pruni[...]
{'fraction_non_alphanumeric': 0.13099817776877912, 'fraction_numerical': 0.06317068232435716, 'mean_word_length': 2.705926481620405, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '547502', 'license': 'MIT', 'n_tokens_mistral': 1977, 'n_tokens_neox': 1823, 'n_words': 321, 'path': 'Boda/parallel models/XGB.ipynb', 'repo_name': 'UVA-DSI-2019-Capstones/UVACyber', 'size': '4939'}