<jupyter_start><jupyter_text>Load vectorizers<jupyter_code>infiles = ['parted-cls/orders_vectorizer30.joblib.pkl.0']

import os
from sklearn.externals import joblib
from lib.helpers.fileSplitter import cat_files

cat_files(infiles, 'cls/orders_vectorizer30.joblib.pkl')

filename = 'cls/orders_vectorizer30.joblib.pkl'
orders_vectorizer = joblib.load(filename)

infiles = ['parted-cls/values_vectorizer30.joblib.pkl.0']

cat_files(infiles, 'cls/values_vectorizer30.joblib.pkl')

filename = 'cls/values_vectorizer30.joblib.pkl'
values_vectorizer = joblib.load(filename)<jupyter_output><empty_output><jupyter_text>Load classifyers<jupyter_code>infiles = ['parted-cls/device_class_code_logreg_cls.joblib.pkl.0']

cat_files(infiles, 'cls/device_class_code_logreg_cls.joblib.pkl')
clf_device_class_code = joblib.load('cls/device_class_code_logreg_cls.joblib.pkl')

infiles = ['parted-cls/os_family_code_logreg_cls.joblib.pkl.0']

cat_files(infiles, 'cls/os_family_code_logreg_cls.joblib.pkl')
clf_os_family_code = joblib.load('cls/os_family_code_logreg_cls.joblib.pkl')

infiles = ['parted-cls/os_code_logreg_cls.joblib.pkl.0']

cat_files(infiles, 'cls/os_code_logreg_cls.joblib.pkl')
clf_os_code = joblib.load('cls/os_code_logreg_cls.joblib.pkl')

infiles = ['parted-cls/ua_class_code_logreg_cls.joblib.pkl.0']

cat_files(infiles, 'cls/ua_class_code_logreg_cls.joblib.pkl')
clf_ua_class_code = joblib.load('cls/ua_class_code_logreg_cls.joblib.pkl')

infiles = ['parted-cls/ua_family_code_logreg_cls.joblib.pkl.0']

cat_files(infiles, 'cls/ua_family_code_logreg_cls.joblib.pkl')
clf_ua_family_code = joblib.load('cls/ua_family_code_logreg_cls.joblib.pkl')

infiles = ['parted-cls/ua_version_logreg_cls.joblib.pkl.0']

cat_files(infiles, 'cls/ua_version_logreg_cls.joblib.pkl')
clf_ua_version = joblib.load('cls/ua_version_logreg_cls.joblib.pkl')<jupyter_output><empty_output><jupyter_text>Load test data<jupyter_code>main_data = np.load('df/main2_data.npy').tolist()[:20000]
values_data = np.load('df/values2_data.npy').tolist()[:20000]
order_data = np.load('df/order2_data.npy').tolist()[:20000]

main_df = pd.DataFrame(main_data)

main_df
important_values_keys_set = {
    'Accept', 
    'Accept-Charset', 
    'Accept-Encoding'
}
important_orders_keys_set = {
    'Upgrade-Insecure-Requests',
    'Accept', 
    'If-Modified-Since',
    'Host', 
    'Connection', 
    'User-Agent', 
    'From', 
    'Accept-Encoding' 
}

from lib.parsers.logParser import LogParser

l_parser = LogParser(log_folder='Logs/')

l_parser.reassign_orders_values(order_data, values_data)
X_test = l_parser.prepare_data(orders_vectorizer, values_vectorizer, important_orders_keys_set, important_values_keys_set, fit_dict=False)<jupyter_output>100%|██████████| 20000/20000 [00:00<00:00, 147657.90it/s]
100%|██████████| 20000/20000 [00:00<00:00, 84219.67it/s]
100%|██████████| 20000/20000 [00:00<00:00, 230988.68it/s]<jupyter_text>Calculate scoresПримечание: Для Decision Tree в cross_val_score по умолчанию берется показатель 'Accuracy'Поскольку 'Accuracy' для линейной регрессии линейный мы не будем считать на 3-х или 5-ти фолдах(долго), а просто возьмем от тренировочной выборки 'Accuracy'<jupyter_code>thres = 0.01<jupyter_output><empty_output><jupyter_text>**Browser (clf_ua_family_code)**<jupyter_code>from lib.thresholdPredictions import ThresholdPredictions

pred = ThresholdPredictions(user_agent_list=clf_ua_family_code.classes_.tolist(), clf=clf_ua_family_code)
y_test_names, y_predicted, compare_answers, answers_count = pred.predict(X_test, main_df.ua_family_code, thres, sparce_y=False, single_labels=True)
compare_frame = pd.concat(
    [
        pd.DataFrame(y_test_names),
        y_predicted, 
        pd.DataFrame(compare_answers),  
        pd.DataFrame(answers_count)
    ], keys=['browser_name', 'predicted_browser_name', 'browser_name_correctness', 'browser_name_count'], axis=1, join='inner')

compare_frame<jupyter_output><empty_output><jupyter_text>Accuracy: $ACC = \frac{TP + TN}{P + N},\ \ \mathrm{where}\ \ P + N = length,\ \ TP = sum(True), \ \ TN = 0$<jupyter_code>print('Accuracy: {}'.format(sum(compare_frame.browser_name_correctness[0])/20000))<jupyter_output>Accuracy: 0.96125<jupyter_text>**Browser + Browser version (clf_ua_family_code + clf_ua_version)**<jupyter_code>pred = ThresholdPredictions(user_agent_list=clf_ua_version.classes_.tolist(), clf=clf_ua_version)
y_test_names, y_predicted, compare_answers, answers_count = pred.predict(X_test, main_df.ua_version, thres, sparce_y=False, single_labels=True)
compare_frame['browser_version'] = pd.DataFrame(y_test_names)
compare_frame['predicted_browser_version'] = y_predicted
compare_frame['browser_version_correctness'] = pd.DataFrame(compare_answers)
compare_frame['browser_version_count'] = pd.DataFrame(answers_count)
compare_frame
print('Browser version accuracy: {}'.format(sum(compare_frame.browser_version_correctness)/20000))
print('Conditional accuracy: {}'.format(sum(compare_frame.browser_name_correctness[0] & compare_frame.browser_version_correctness)/20000))<jupyter_output>Browser version accuracy: 0.8155
Conditional accuracy: 0.8141<jupyter_text>**Browser + Browser version + Platform (clf_ua_family_code + clf_ua_version + clf_os_family_code)**<jupyter_code>pred = ThresholdPredictions(user_agent_list=clf_os_family_code.classes_.tolist(), clf=clf_os_family_code)
y_test_names, y_predicted, compare_answers, answers_count = pred.predict(X_test, main_df.os_family_code, thres, sparce_y=False, single_labels=True)
compare_frame['platform'] = pd.DataFrame(y_test_names)
compare_frame['predicted_platform'] = y_predicted
compare_frame['platform_correctness'] = pd.DataFrame(compare_answers)
compare_frame['platform_count'] = pd.DataFrame(answers_count)
compare_frame
print('Platform accuracy: {}'.format(sum(compare_frame.platform_correctness)/20000))
print('Conditional accuracy: {}'.format(sum(compare_frame.browser_name_correctness[0] & compare_frame.browser_version_correctness & compare_frame.platform_correctness)/20000))<jupyter_output>Platform accuracy: 0.9648
Conditional accuracy: 0.80905<jupyter_text>**Browser + Browser version + Platform + Platform version (clf_ua_family_code + clf_ua_version + clf_os_family_code + clf_os_code)**<jupyter_code>pred = ThresholdPredictions(user_agent_list=clf_os_code.classes_.tolist(), clf=clf_os_code)
y_test_names, y_predicted, compare_answers, answers_count = pred.predict(X_test, main_df.os_code, thres, sparce_y=False, single_labels=True)
compare_frame['platform_version'] = pd.DataFrame(y_test_names)
compare_frame['predicted_platform_version'] = y_predicted
compare_frame['platform_version_correctness'] = pd.DataFrame(compare_answers)
compare_frame['platform_version_count'] = pd.DataFrame(answers_count)
compare_frame
print('Platform accuracy: {}'.format(sum(compare_frame.platform_version_correctness)/20000))
print('Conditional accuracy: {}'.format(sum(compare_frame.browser_name_correctness[0] & compare_frame.browser_version_correctness & compare_frame.platform_correctness & compare_frame.platform_version_correctness)/20000))<jupyter_output>Platform accuracy: 0.9445
Conditional accuracy: 0.79785<jupyter_text>Save and load compare dataframe<jupyter_code>np.save('df/compare_dummy_classification.npy', np.array(compare_frame))
print("Compare frame was saved with size(Bytes): {}".format(os.stat('df/compare_dummy_classification.npy').st_size))
compare_frame2 = pd.DataFrame(np.load('df/compare_dummy_classification.npy').tolist())
compare_frame2.columns = [
    'browser_name', 
    'predicted_browser_name', 
    'browser_name_correctness', 
    'browser_name_count', 
    'browser_version', 
    'predicted_browser_version', 
    'browser_version_correctness', 
    'browser_version_count', 
    'platform', 
    'predicted_platform', 
    'platform_correctness', 
    'platform_count',
    'platform_version', 
    'predicted_platform_version', 
    'platform_version_correctness', 
    'platform_version_count'
]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12567280010013768, 'fraction_numerical': 0.026286143447239955, 'mean_word_length': 8.411071849234393, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '544268', 'license': 'MIT', 'n_tokens_mistral': 3120, 'n_tokens_neox': 2947, 'n_words': 339, 'path': '10-GroupPredictionComparsion.ipynb', 'repo_name': 'levabd/http-ml-antifraud', 'size': '7989'}