<jupyter_start><jupyter_text>One-hot encoding<jupyter_code>train_data = dict()
for family in family_dirs:
    traceFiles_train = list()
    for f in os.listdir(family):
        if not os.path.isdir(f):
            traceFiles_train.append(family+f)
    train_data[family] = traceFiles_train
train_data['11939data/top3_party_thesis_train/allaple/'][0]
colCandidates = set()
# for familyName, tracePaths in train_data.items():
#     featureTraces = getFeatureProfiles(tracePaths)
#     for sha256 in featureTraces.keys():
#         trace = featureTraces[sha256]
#         colCandidates.update(trace)
# print(len(colCandidates))
import pickle
# with open("data/featured_api_dimen_28family_thesis.pkl", 'wb') as f:
#     pickle.dump(colCandidates, f)

with open("data/featured_api_dimen_28family_0629.pkl", 'rb') as f:
    colCandidates = pickle.load(f)
print(len(colCandidates))
df = pd.DataFrame(columns=sorted(list(colCandidates)))
df.index.name = 'sha256'
df.shape<jupyter_output><empty_output><jupyter_text>Encoding Training Data<jupyter_code>for familyName in sorted(train_data.keys()):
    tracePaths = train_data[familyName]
    featureTraces = getFeatureProfiles(tracePaths)
    
    for sha256 in sorted(featureTraces.keys()):
        trace = featureTraces[sha256]

        row = {col:0 for col in colCandidates}
        for api in trace:
            row[api] = 1

        df.loc[sha256] = row
print(df.shape)
df<jupyter_output>(3149, 53184)<jupyter_text>Encoding Testing Data<jupyter_code>testDF = pd.DataFrame(columns=sorted(list(colCandidates)))
testDF.shape
base_dir_test = '11939data/top3_party_thesis_test/'
family_dirs_test = [base_dir_test + f + '/' for f in os.listdir(base_dir_test)]

test_data = dict()
for family in family_dirs_test:
    traceFiles_test = list()
    for f in os.listdir(family):
        if not os.path.isdir(f):
            traceFiles_test.append(family+f)
    test_data[family] = traceFiles_test


for familyName in sorted(test_data.keys()):
    tracePaths = test_data[familyName]
    featureTraces = getFeatureProfiles(tracePaths)
    
    for sha256 in sorted(featureTraces.keys()):
        trace = featureTraces[sha256]
        row = {col:0 for col in colCandidates}
        for api in trace:
            if api in colCandidates:
                row[api] = 1
        testDF.loc[sha256] = row<jupyter_output><empty_output><jupyter_text>Build data mining model with different Classification Algorithms.<jupyter_code>print(testDF.shape)
testDF<jupyter_output>(410, 53184)<jupyter_text>Change Dataframe to numpy matrixSklearn use array as input.<jupyter_code>train_labels = []
test_labels = []

for familyName in sorted(train_data.keys()):
    fileCount = 0
    tracePaths = train_data[familyName]
    
    for tracePath in tracePaths:
        if os.path.isfile(tracePath):
            train_labels.append(familyName.split('/')[-2])
            fileCount += 1
            
for familyName in sorted(test_data.keys()):
    fileCount = 0
    tracePaths = test_data[familyName]
    
    for tracePath in tracePaths:
        if os.path.isfile(tracePath):
            test_labels.append(familyName.split('/')[-2])
            fileCount += 1
X_Train = df.as_matrix() # trainin data
Y_Train = np.array(train_labels) # training label
X_Test = testDF.as_matrix() # testing data
Y_Test = np.array(test_labels) # testing label
### create Sample Name Index

sampleIndex = list()
for traceName in df.index:
    sampleName = traceName.split('_')[0]
    sampleIndex.append(sampleName)

### create Sample Label List Index
labelList = sorted(list(set(Y_Train)))
labelIDX = {l:idx for idx, l in enumerate(labelList , start=1)}

# Training set labels
new_Y_Train = []
for ytra in Y_Train:
    new_Y_Train.append(labelIDX[ytra])
new_Y_Train = np.array(new_Y_Train)

# Testing set labels
new_Y_Test = []
for ytra in Y_Test:
    new_Y_Test.append(labelIDX[ytra])
new_Y_Test = np.array(new_Y_Test)
from sklearn.metrics import accuracy_score
from sklearn.metrics import f1_score, precision_score, recall_score
def getSamplePNR(sampleIndex, y_true, y_pred):
    lastSample = ""
    lastTrueLabel = ""
    isMatch = False
    matchResult = dict()
    predList = []
    samplePredDict = dict()
    for idx, pred in enumerate(y_pred):
        currentSample = sampleIndex[idx]
        trueLabel = y_true[idx]
        if currentSample != lastSample:
            if idx != 0:
                samplePredDict[lastSample] = (lastTrueLabel, predList)
                predList = []
                isMatch = False

            if pred == trueLabel:
                matchResult[currentSample] = 'Match'
                isMatch = True
            else:
                matchResult[currentSample] = 'Mismatch'

            lastSample = currentSample
            lastTrueLabel = trueLabel
        else:
            if not isMatch:
                if pred == trueLabel:
                    matchResult[currentSample] = 'Match'
                    isMatch = True

        predList.append(pred)

    samplePredDict[lastSample] = (lastTrueLabel, predList)

    my_true = []
    my_pred = []
    for sha, labels in samplePredDict.items():
        trueLabel, predList = labels
    #     print("==> true:", trueLabel)
    #     print(predList)
        if trueLabel in predList:
            my_pred.append(trueLabel)
        else:
            my_pred.append(predList[0])
        my_true.append(trueLabel)

    print("accuracy rate:",accuracy_score(my_true, my_pred))
    print("--Micro-Average--")
    print('micro precesion:',precision_score(my_true, my_pred, average='micro'))
    print('micro recall:',recall_score(my_true, my_pred, average='micro'))
    print('micro f1:',f1_score(my_true, my_pred, average='micro'))

    print("--Macro-Average--")
    print('macro precesion:',precision_score(my_true, my_pred, average='macro'))
    print('macro recall:',recall_score(my_true, my_pred, average='macro'))
    print('macro f1:',f1_score(my_true, my_pred, average='macro'))   
    
    print("--Weighted-Average--")
    print('weighted precesion:',precision_score(my_true, my_pred, average='weighted'))
    print('weighted recall:',recall_score(my_true, my_pred, average='weighted'))
    print('weighted f1:',f1_score(my_true, my_pred, average='weighted'))   
from sklearn.svm import SVC

svm_classifier = SVC(kernel='rbf', C=100)
svm_classifier.fit(X_Train, new_Y_Train)
svm_pred = svm_classifier.predict(X_Test)

y_true = new_Y_Test
y_pred = svm_pred
print("--SVM--")
getSamplePNR(sampleIndex, y_true, y_pred)
from sklearn.naive_bayes import GaussianNB


gaussianNB_classifier=GaussianNB()
gaussianNB_classifier.fit(X_Train, Y_Train)
gsNB_pred = gaussianNB_classifier.predict(X_Test)

y_true = Y_Test
y_pred = gsNB_pred
print("--Gaussian NB--")
getSamplePNR(sampleIndex, y_true, y_pred)
from sklearn.naive_bayes import BernoulliNB

bernoulliNB_classifier=BernoulliNB()
bernoulliNB_classifier.fit(X_Train, Y_Train)
bnNB_pred = bernoulliNB_classifier.predict(X_Test)

y_true = Y_Test
y_pred = bnNB_pred
print("--Bernoulli NB--")
getSamplePNR(sampleIndex, y_true, y_pred)
from sklearn import neighbors
n_neighbors = 5
knn_classifier = neighbors.KNeighborsClassifier(n_neighbors)
knn_classifier.fit(X_Train, Y_Train)
knn_pred5 = knn_classifier.predict(X_Test)

y_true = Y_Test
y_pred = knn_pred5
print("--KNN--")
getSamplePNR(sampleIndex, y_true, y_pred)
from sklearn.tree import DecisionTreeClassifier

dt_classifier = DecisionTreeClassifier()
dt_classifier.fit(X_Train, Y_Train)
dt_pred = dt_classifier.predict(X_Test)

y_true = Y_Test
y_pred = dt_pred
print("--Decision Tree--")
getSamplePNR(sampleIndex, y_true, y_pred)<jupyter_output>--Decision Tree--
accuracy rate: 0.593582887701
--Micro-Average--
micro precesion: 0.593582887701
micro recall: 0.593582887701
micro f1: 0.593582887701
--Macro-Average--
macro precesion: 0.552238770002
macro recall: 0.518258774323
macro f1: 0.506453246484
--Weighted-Average--
weighted precesion: 0.6004607106
weighted recall: 0.593582887701
weighted f1: 0.559989527313
{'fraction_non_alphanumeric': 0.11084699111722758, 'fraction_numerical': 0.027899411985487302, 'mean_word_length': 3.8595744680851065, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '75421', 'license': 'MIT', 'n_tokens_mistral': 2966, 'n_tokens_neox': 2710, 'n_words': 537, 'path': 'Traditional_method_classifying.ipynb', 'repo_name': 'weichih-c/Hooklog-Analysis', 'size': '7993'}