<jupyter_start><jupyter_text>\ Developer: <NAME> (<EMAIL>) \ 5th March 2019 \ © 2019 initOS GmbH \ License MIT \ Library for TSVM and SelfLearning taken from https://github.com/tmadl/semisup-learn \ Library for lagrangean-S3VM taken from https://github.com/fbagattini/lagrangean-s3vm<jupyter_code>from sklearn.svm import SVC
import pandas as pd
import numpy as np
from __future__ import division
import re
from sklearn.model_selection import StratifiedShuffleSplit
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from frameworks.SelfLearning import *
from imblearn.over_sampling import SMOTE
from collections import Counter
from imblearn.under_sampling import RepeatedEditedNearestNeighbours
from sklearn.naive_bayes import MultinomialNB
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import GridSearchCV
from sklearn.externals import joblib
import time
import matplotlib.pyplot as plt
from sklearn.metrics import classification_report
from methods.scikitTSVM import SKTSVM
from sklearn.metrics import confusion_matrix
from sklearn.utils.multiclass import unique_labels
source_domain = pd.read_csv('github_preprocessed_data.csv')
target_domain = pd.read_csv('mantis_data_for_domain_adaptation.csv')
source_domain.drop(["Unnamed: 0"], axis=1, inplace=True)
target_domain.drop(["Unnamed: 0"], axis=1, inplace=True)

source_domain['text'] = source_domain['text'].fillna("")
target_domain['textual_data'] = target_domain['textual_data'].fillna("")
print source_domain['type'].value_counts()
print target_domain['type'].value_counts()
unlabelled_index_target = target_domain[(target_domain['bug_or_not'].isnull())].index
labelled_index_target = target_domain[~(target_domain['bug_or_not'].isnull())].index
len(unlabelled_index_target), len(labelled_index_target)
target_domain_labeled = target_domain.loc[labelled_index_target]
len(target_domain_labeled), len(target_domain)
tfidf_vectorizer_source = TfidfVectorizer(max_df=0.95, min_df=2, max_features=500, stop_words='english')
source_domain_balanced = source_domain.groupby('type').apply(lambda x: x.sample(400))
print source_domain['type'].value_counts()
source_domain_X = tfidf_vectorizer_source.fit_transform(source_domain_balanced['text'])
source_domain_Y = np.array(source_domain_balanced['type'])
stratified_shuffle_split = StratifiedShuffleSplit(n_splits=3, test_size=0.3, random_state=0)
scores = []
iteration = 1
for train_index, test_index in stratified_shuffle_split.split(source_domain_X, source_domain_Y):
    X_train = source_domain_X[train_index].copy()
    Y_train = source_domain_Y[train_index].copy()
    X_test = source_domain_X[test_index].copy()
    Y_test = source_domain_Y[test_index].copy()
    clf = MultinomialNB()
    clf.fit(X_train, Y_train)
    print clf.score(X_test, Y_test.astype(float))
    y_pred = clf.predict(X_test)
    result = classification_report(Y_test.astype(float), y_pred.astype(float), output_dict=True)
    src = pd.DataFrame(result)
    src.transpose().to_csv('{}_{}_{}_latex_table_report.csv'.format('source_vs_source', '500', iteration))
    print src.transpose()
    iteration += 1<jupyter_output>0.845833333333
              f1-score  precision    recall  support
0.0           0.841202   0.867257  0.816667    120.0
1.0           0.850202   0.826772  0.875000    120.0
macro avg     0.845702   0.847014  0.845833    240.0
micro avg     0.845833   0.845833  0.845833    240.0
weighted avg  0.845702   0.847014  0.845833    240.0
0.85
              f1-score  precision    recall  support
0.0           0.848739   0.855932  0.841667    120.0
1.0           0.851240   0.844262  0.858333    120.0
macro avg     0.849990   0.850097  0.850000    240.0
micro avg     0.850000   0.850000  0.850000    240.0
weighted avg  0.849990   0.850097  0.850000    240.0
0.895833333333
              f1-score  precision    recall  support
0.0           0.889868   0.943925  0.841667    120.0
1.0           0.901186   0.857143  0.950000    120.0
macro avg     0.895527   0.900534  0.895833    240.0
micro avg     0.895833   0.895833  0.895833    240.0
weighted avg  0.895527   0.900534  0.895833    240.0<jupyter_text>Baseline TL Source VS Target Supervised<jupyter_code>source_domain_X = tfidf_vectorizer_source.fit_transform(source_domain['text'])
source_domain_Y = np.array(source_domain['type'])
for x in xrange(5):
    clf = MultinomialNB()
    clf.fit(source_domain_X, source_domain_Y)
    target_domain_labeled_balanced = target_domain_labeled.groupby('type').apply(lambda x: x.sample(90))
    target_domain_X = tfidf_vectorizer_source.transform(target_domain_labeled_balanced['textual_data'])
    target_domain_Y = np.array(target_domain_labeled_balanced['type'])
    print("members for classes {}".format(",".join("(%s,%s)" % tup for tup in sorted(Counter(target_domain_Y).items()))))
    score = clf.score(target_domain_X, target_domain_Y.astype(float))
    print "Baseline TL Score: "+ str(score)
    y_pred = clf.predict(target_domain_X)
    print("members for classes {}".format(",".join("(%s,%s)" % tup for tup in sorted(Counter(y_pred).items()))))
    result = classification_report(target_domain_Y.astype(float), y_pred.astype(float), output_dict=True)
    src = pd.DataFrame(result)
    print src
    src.transpose().to_csv('{}_{}_{}_latex_table_report.csv'.format('source_vs_target_supervised', '500', x))<jupyter_output>members for classes (0.0,90),(1.0,90)
Baseline TL Score: 0.566666666667
members for classes (0,44),(1,136)
                 0.0        1.0   macro avg   micro avg  weighted avg
f1-score    0.417910   0.654867    0.536389    0.566667      0.536389
precision   0.636364   0.544118    0.590241    0.566667      0.590241
recall      0.311111   0.822222    0.566667    0.566667      0.566667
support    90.000000  90.000000  180.000000  180.000000    180.000000
members for classes (0.0,90),(1.0,90)
Baseline TL Score: 0.6
members for classes (0,38),(1,142)
                 0.0        1.0   macro avg  micro avg  weighted avg
f1-score    0.437500   0.689655    0.563578        0.6      0.563578
precision   0.736842   0.563380    0.650111        0.6      0.650111
recall      0.311111   0.888889    0.600000        0.6      0.600000
support    90.000000  90.000000  180.000000      180.0    180.000000
members for classes (0.0,90),(1.0,90)
Baseline TL Score: 0.6
members for classes (0,36),(1,144)
      [...]<jupyter_text>TL Source Semi-Supervised<jupyter_code>target_domain_unlabeled = target_domain.loc[unlabelled_index_target, ["textual_data", "type"]].copy()
target_domain_unlabeled["type"] = -1
source_domain_df = source_domain[["text", "type"]].copy()
source_domain_df.rename({"text":"textual_data", "type":"type"}, axis=1, inplace=1)
domain_adaptation_df = pd.concat([source_domain_df, target_domain_unlabeled])
len(domain_adaptation_df), len(source_domain_df), len(target_domain_unlabeled)
print domain_adaptation_df['type'].value_counts()
print target_domain_labeled_balanced['type'].value_counts()
from collections import Counter
#print("members for classes {}".format(",".join("(%s,%s)" % tup for tup in sorted(Counter(Y).items()))))
def domain_adaptation(dom_a_df, target_domain_labeled, 
                                        classifier, label_type, neg_class, classifier_name):
    dom_a_df.loc[dom_a_df['type']==0, 'type'] = neg_class
    dom_a_df.loc[dom_a_df['type']==1, 'type'] = 1
    
    target_domain_labeled.loc[target_domain_labeled['type']==0, 'type'] = neg_class
    target_domain_labeled.loc[target_domain_labeled['type']==1, 'type'] = 1
    
    tfidf_vectorizer_source = TfidfVectorizer(max_df=0.95, min_df=2, max_features=500, stop_words='english')
    source_X = tfidf_vectorizer_source.fit_transform(dom_a_df['textual_data']).toarray()
    source_Y = np.array(dom_a_df['type'])
    
    target_domain_X = tfidf_vectorizer_source.transform(target_domain_labeled['textual_data']).toarray()
    target_domain_Y = np.array(target_domain_labeled['type'])
    
    if label_type != 'int':
        source_Y = source_Y.astype(float)
    else:
         source_Y = source_Y.astype(int)
            
    classifier.fit(source_X, source_Y)
    score = classifier.score(target_domain_X, target_domain_Y.astype(int))
    joblib.dump(classifier, 'models/DA_{}.pkl'.format(classifier_name))
    joblib.dump(target_domain_X, 'models/X_test_DA_{}.pkl'.format(classifier_name))
    joblib.dump(target_domain_Y, 'models/Y_test_DA_{}.pkl'.format(classifier_name))
    print "{} score: {}".format(classifier_name, score)
sklearn_lr = LogisticRegression(solver='lbfgs')
domain_adaptation(domain_adaptation_df.copy(), target_domain_labeled_balanced.copy(), 
                               SelfLearningModel(sklearn_lr), 'float', 0, 'ST_LR')
domain_adaptation(domain_adaptation_df.copy(), target_domain_labeled_balanced.copy(), 
                               SKTSVM(), 'int', 0, 'TSVM')
def plot_confusion_matrix(y_true, y_pred, classes,
                          normalize=False,
                          title=None,
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    if not title:
        if normalize:
            title = 'Normalized confusion matrix'
        else:
            title = 'Confusion matrix, without normalization'

    # Compute confusion matrix
    cm = confusion_matrix(y_true, y_pred)
    # Only use the labels that appear in the data
    #classes = classes[unique_labels(y_true, y_pred)]
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')

    print(cm)

    fig, ax = plt.subplots()
    im = ax.imshow(cm, interpolation='nearest', cmap=cmap)
    ax.figure.colorbar(im, ax=ax)
    # We want to show all ticks...
    ax.set(xticks=np.arange(cm.shape[1]),
           yticks=np.arange(cm.shape[0]),
           # ... and label them with the respective list entries
           xticklabels=['Bug', 'Non-Bug'], yticklabels=['Bug', 'Non-Bug'],
           title=title,
           ylabel='True label',
           xlabel='Predicted label')

    # Rotate the tick labels and set their alignment.
    plt.setp(ax.get_xticklabels(), rotation=45, ha="right",
             rotation_mode="anchor")

    # Loop over data dimensions and create text annotations.
    fmt = '.2f' if normalize else 'd'
    thresh = cm.max() / 2.
    for i in range(cm.shape[0]):
        for j in range(cm.shape[1]):
            ax.text(j, i, format(cm[i, j], fmt),
                    ha="center", va="center",
                    color="white" if cm[i, j] > thresh else "black")
    fig.tight_layout()
    #plt.savefig('Confusion_matrix(Phase_3).png', bbox_inches='tight')
    return ax
def get_results(classifier, data_type):
    dict_features = {}
    model = joblib.load('models/DA_{}.pkl'.format(classifier, 3))
    x_tst = joblib.load('models/X_test_DA_{}.pkl'.format(classifier, 3))
    y_tst = joblib.load('models/Y_test_DA_{}.pkl'.format(classifier, 3))
    acc = model.score(x_tst, y_tst.astype(data_type))
    print acc
    y_pred = model.predict(x_tst)
    print("members for classes {}".format(",".join("(%s,%s)" % tup for tup in sorted(Counter(y_tst).items()))))
    print("members for classes {}".format(",".join("(%s,%s)" % tup for tup in sorted(Counter(y_pred).items()))))
    result = classification_report(y_tst.astype(data_type), y_pred.astype(data_type), output_dict=True)
    result_df = pd.DataFrame(result)
    result_df.transpose().to_csv('DA_{}_latex_table_report.csv'.format(classifier))
    np.set_printoptions(precision=2)

    # Plot non-normalized confusion matrix
    plot_confusion_matrix(y_tst.astype(data_type), y_pred.astype(data_type), classes=[0, 1],
                          title='Confusion matrix, without normalization')

    # Plot normalized confusion matrix
    #plot_confusion_matrix(y_tst.astype(data_type), y_pred.astype(data_type), classes=[0, 1], normalize=True,
     #                     title='Normalized confusion matrix')

    plt.show()
    print result_df.transpose()
    return result_df
st_results = get_results('ST_LR', float)
tsvm_results = get_results('TSVM', int)<jupyter_output>0.516666666667
members for classes (0.0,90),(1.0,90)
members for classes (0.0,5),(1.0,175)
Confusion matrix, without normalization
[[ 4 86]
 [ 1 89]]
{'fraction_non_alphanumeric': 0.1225588684400868, 'fraction_numerical': 0.0705617616330467, 'mean_word_length': 3.8590394377196406, 'pattern_counts': {'":': 2, '<': 12, '<?xml version=': 0, '>': 13, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '1177', 'license': 'MIT', 'n_tokens_mistral': 4835, 'n_tokens_neox': 4258, 'n_words': 818, 'path': '9) domain_adaptation/5.domain_adaptation.ipynb', 'repo_name': 'initOS/research-criticality-identification', 'size': '12463'}