<jupyter_start><jupyter_text>TODO* Precision-Recall http://scikit-learn.org/stable/auto_examples/model_selection/plot_precision_recall.html* Use Git meta data as feature  Columns0. id1. repository_id2. blamed_commit_id3. type4. sha5. url6. author_email7. author_name8. author_when9. committer_email10. committer_name11. committer_when12. additions13. deletions14. total_changes15. past_changes16. future_changes17. past_different_authors18. future_different_authors19. author_contributions_percent20. message21. patch22. hunk_count23. cve24. files_changed25. patch_keywords<jupyter_code>import psycopg2
conn = psycopg2.connect("dbname=postgres host=localhost port=55432 user=postgres")
cur = conn.cursor()
cur.execute("SELECT type, COUNT(id) FROM export.commits GROUP BY type")
cur.fetchall()

import matplotlib.pyplot as plt
from sklearn.metrics import precision_recall_curve
from sklearn.metrics import average_precision_score
from sklearn.cross_validation import train_test_split
from sklearn.svm import LinearSVC
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.preprocessing import label_binarize
from sklearn import cross_validation
import numpy as np
import scipy as sp
import enum
from datetime import datetime
from diff_extractor import extract_lines, extract_added_lines, extract_removed_lines

class Colum(enum.IntEnum):
    id = 1
    type = 3
    additions = 13
    deletions = 14
    past_different_authors = 18
    future_different_authors = 19
    author_contributions_percent = 20
    patch = 21
    hunk_count = 23
    files_changed = 25
    
def fetch(filename, key = None):
    npz = np.load(filename)
    key = npz.files[0] if key is None else key
    data = npz[key]
    npz.close()
    return data


# data = fetch('var/vcc_sample_40x800.npz')
data = fetch('var/vcc_data.npz')

patches = data[:, Colum.patch]
labels = data[:, Colum.type]

# Note: Be sure that unicode(patch, 'utf-8')
tokens = [u" ".join(extract_lines(patch.splitlines())) for patch in patches]
print tokens[-1]

# vectorizer = CountVectorizer(min_df=1)
# vectorized = vectorizer.fit_transform(text)
# # feature_names = vectorizer.get_feature_names()
# X = vectorized.toarray()

# y = is_vcc = (labels == 'blamed_commit')

# # Metrics from Git metadata
# additions = data[:, Colum.additions]
# deletions = data[:, Colum.deletions]
# pd_authors = data[:, Colum.past_different_authors]
# fd_authors = data[:, Colum.future_different_authors]
# ac_rate = data[:, Colum.author_contributions_percent]
# hunk_count = data[:, Colum.hunk_count]
# files_changed = data[:, Colum.files_changed]

# TODO
# https://docs.scipy.org/doc/numpy-1.9.1/reference/generated/numpy.log.html

# # Split into training and test
# X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.5, random_state=0)

# # Run classifier
# classifier = LinearSVC(C=1.0)
# y_score = classifier.fit(X_train, y_train).decision_function(X_test)

# # Compute Precision-Recall and plot curve
# precision = dict()
# recall = dict()
# average_precision = dict()
# precision[0], recall[0], _ = precision_recall_curve(y_test, y_score)
# average_precision[0] = average_precision_score(y_test, y_score)

# print precision, recall, average_precision

# # Plot Precision-Recall curve
# plt.clf()
# plt.plot(recall[0], precision[0], label='Precision-Recall curve')
# plt.xlabel('Recall')
# plt.ylabel('Precision')
# plt.ylim([0.0, 1.05])
# plt.xlim([0.0, 1.0])
# plt.title('Precision-Recall example: AUC={0:0.2f}'.format(average_precision[0]))
# plt.legend(loc="lower left")
# plt.show()

# plt.savefig("figure_%s" % datetime.now().strftime('%s'))

import psycopg2
import psycopg2.extensions

fp = open('6b741be98ed571cd8e18d68df88c25e61fc87305.patch')
patch = fp.read()
fp.close()

conn = psycopg2.connect("dbname=postgres host=localhost port=55432 user=postgres")
cursor = conn.cursor()
cursor.execute("UPDATE export.commits SET patch=(%s) WHERE id = (%s)", (patch,12315129,));
conn.commit()
cursor.close()
# Cleansing invalid patch
# E.g.,
# UnidiffParseError: Hunk diff line expected: index 4fdc414..44faea4 100644
import numpy as np
import scipy as sp
import enum
import psycopg2
import psycopg2.extensions
from datetime import datetime
from diff_extractor import extract_lines, extract_added_lines, extract_removed_lines
from unidiff.errors import UnidiffParseError

psycopg2.extensions.register_type(psycopg2.extensions.UNICODE)
psycopg2.extensions.register_type(psycopg2.extensions.UNICODEARRAY)

class Colum(enum.IntEnum):
    id = 1
    type = 3
    additions = 13
    deletions = 14
    past_different_authors = 18
    future_different_authors = 19
    author_contributions_percent = 20
    patch = 21
    hunk_count = 23
    files_changed = 25
    
    
class Data:
    def __init__(self, config):
        self.cur = psycopg2.connect(config).cursor()
    def fetch(self, sql):
        self.cur.execute(sql)
        return self.cur.fetchall()

data = Data("dbname=postgres host=localhost port=55432 user=postgres")
rows = np.array(data.fetch("SELECT * FROM export.commits -- WHERE id >= 12315129"))
patches = rows[:, Colum.patch]

print 'Started parsing patches...'
patch = [None]
try:
    cleansed_paches = [extract_lines(patch.splitlines()) for patch in patches]
except UnidiffParseError as e:
    print patch
    raise e

print 'Completed cleansing!'<jupyter_output>Started parsing patches...
diff --git a/include/libvirt/libvirt.h.in b/include/libvirt/libvirt.h.in
index 4eae7bf..7bd3559 100644
--- a/include/libvirt/libvirt.h.in
+++ b/include/libvirt/libvirt.h.in
@@ -181,7 +181,7 @@ typedef enum {
     VIR_DOMAIN_PAUSED_FROM_SNAPSHOT = 7, /* paused after restoring from snapshot */
     VIR_DOMAIN_PAUSED_SHUTTING_DOWN = 8, /* paused during shutdown process */
     VIR_DOMAIN_PAUSED_SNAPSHOT = 9,      /* paused while creating a snapshot */
-    VIR_DOMAIN_PAUSED_GUEST_PANICKED = 10, /* paused due to a guest panicked event */
+    VIR_DOMAIN_PAUSED_CRASHED = 10,     /* paused due to a guest crash */
 
 #ifdef VIR_ENUM_SENTINELS
     VIR_DOMAIN_PAUSED_LAST
diff --git a/src/qemu/qemu_monitor.c b/src/qemu/qemu_monitor.c
index 0b73411..5b2fb04 100644
--- a/src/qemu/qemu_monitor.c
+++ b/src/qemu/qemu_monitor.c
@@ -3335,7 +3335,7 @@ int qemuMonitorVMStatusToPausedReason(const char *status)
         return VIR_DOMAIN_PAUSED_WATCHDOG;
 
     case QEMU_MONITOR[...]
{'fraction_non_alphanumeric': 0.11296412838263059, 'fraction_numerical': 0.03886091881686595, 'mean_word_length': 4.800182481751825, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '38988', 'license': 'MIT', 'n_tokens_mistral': 2397, 'n_tokens_neox': 2210, 'n_words': 490, 'path': 'VCC-Note-Cleansing.ipynb', 'repo_name': 'ymkjp/hvd', 'size': '6356'}