<jupyter_start><jupyter_text>Another User Agent representation User Agent as tuple From Udger `UserAgent = {ua_family_code, ua_version, ua_class_code, device_class_code, os_family_code, os_code}` Analise data<jupyter_code>from lib.parsers.logParser import LogParser

l_parser = LogParser(log_folder='Logs/')

main_data, values_data, order_data = l_parser.parse_train_sample(0, 30, filter_crawlers=True, parse_ua=True)
len(main_data)
main_df = pd.DataFrame(main_data)

main_df<jupyter_output><empty_output><jupyter_text>Funny that row count on first 30 logs equal row count on first 10 logs without filtering !60% logs data was from crowlers!<jupyter_code>list_device_class_code = main_df.device_class_code.value_counts().index.tolist()

list_device_class_code
list_os_family_code = main_df.os_family_code.value_counts().index.tolist()

print("Device platform family count: {}".format(len(list_os_family_code)))
list_os_family_code
list_os_code = main_df.os_code.value_counts().index.tolist()

print("Device platform count: {}".format(len(list_os_code)))
list_os_code
list_ua_class_code = main_df.ua_class_code.value_counts().index.tolist()

print("Device browser class count: {}".format(len(list_ua_class_code)))
list_ua_class_code
list_ua_family_code = main_df.ua_family_code.value_counts().index.tolist()

print("Device browser family count: {}".format(len(list_ua_family_code)))
list_ua_family_code<jupyter_output>Device browser family count: 255<jupyter_text>As we see not all legal crawler was filtered. So we need new filter<jupyter_code>list_ua_version = main_df.ua_version.value_counts().index.tolist()

print("Device browser version count: {}".format(len(list_ua_version)))
list_ua_version<jupyter_output>Device browser version count: 3980<jupyter_text>As we see browser version must be merged with browser code  Another data crowler filter<jupyter_code>%%time

from lib.parsers.logParser import LogParser

l_parser = LogParser(log_folder='Logs/')

main_data, values_data, order_data = l_parser.parse_train_sample(0, 1, filter_crawlers=True, parse_ua=True)

print("Main data length: {}".format(len(main_data)))

main_df = pd.DataFrame(main_data)

main_df
list_ua_family_code = main_df.ua_family_code.value_counts().index.tolist()

print("Device browser family count: {}".format(len(list_ua_family_code)))
list_ua_family_code
%%time

from lib.parsers.logParser import LogParser

l_parser = LogParser(log_folder='Logs/')

main_data, values_data, order_data = l_parser.parse_train_sample(0, 30, filter_crawlers=True, parse_ua=True)

print("Main data length: {}".format(len(main_data)))
main_df = pd.DataFrame(main_data)

main_df

list_device_class_code = pd.DataFrame(main_data).device_class_code.value_counts().index.tolist()
list_os_family_code = pd.DataFrame(main_data).os_family_code.value_counts().index.tolist()
list_os_code = pd.DataFrame(main_data).os_code.value_counts().index.tolist()
list_ua_class_code = pd.DataFrame(main_data).ua_class_code.value_counts().index.tolist()
list_ua_family_code = pd.DataFrame(main_data).ua_family_code.value_counts().index.tolist()
list_ua_version = pd.DataFrame(main_data).ua_version.value_counts().index.tolist()

print("Device count: {}".format(len(list_device_class_code)))
print("Device platform family count: {}".format(len(list_os_family_code)))
print("Device platform count: {}".format(len(list_os_code)))
print("Device browser class count: {}".format(len(list_ua_class_code)))
print("Device browser family count: {}".format(len(list_ua_family_code)))
print("Device browser version count: {}".format(len(list_ua_version)))
list_device_class_code
list_os_family_code
list_os_code
list_ua_class_code
list_ua_family_code
print(type(main_data))
print(type(values_data))
print(type(order_data))<jupyter_output><class 'list'>
<class 'list'>
<class 'list'><jupyter_text>Save data<jupyter_code>np.save('df/main_data.npy', np.array(main_data))
print("Main_data saved with size(Bytes): {}".format(os.stat('df/main_data.npy').st_size))
np.save('df/values_data.npy', np.array(values_data))
print("Values_data saved with size(Bytes): {}".format(os.stat('df/values_data.npy').st_size))
np.save('df/order_data.npy', np.array(order_data))
print("Order_data saved with size(Bytes): {}".format(os.stat('df/order_data.npy').st_size))<jupyter_output>Main_data saved with size(Bytes): 174135401
Values_data saved with size(Bytes): 493202751
Order_data saved with size(Bytes): 80283456<jupyter_text>Bot Data<jupyter_code>from lib.parsers.logParser import LogParser

l_test_parser = LogParser(log_folder='Logs/')

main_bot_data, values_bot_data, order_bot_data = l_test_parser.parse_bot_sample(30, 60, 60, 90, filter_crawlers=True, parse_ua=True)<jupyter_output>0%|          | 0/30 [00:00<?, ?it/s]<jupyter_text>Save data<jupyter_code>np.save('df/main_bot_data.npy', np.array(main_bot_data))
print("Main_bot_data saved with size(Bytes): {}".format(os.stat('df/main_bot_data.npy').st_size))
np.save('df/values_bot_data.npy', np.array(values_bot_data))
print("Values_bot_data saved with size(Bytes): {}".format(os.stat('df/values_bot_data.npy').st_size))
np.save('df/order_bot_data.npy', np.array(order_bot_data))
print("Order_bot_data saved with size(Bytes): {}".format(os.stat('df/order_bot_data.npy').st_size))<jupyter_output>Main_bot_data saved with size(Bytes): 115773166
Values_bot_data saved with size(Bytes): 620766475
Order_bot_data saved with size(Bytes): 92914382<jupyter_text>Test human data<jupyter_code>l_parser_tmp = LogParser(log_folder='Logs/')

main_test_data, values_test_data, order_test_data = l_parser_tmp.parse_train_sample(90, 120, filter_crawlers=True, parse_ua=True)

print("Main test data length: {}".format(len(main_test_data)))
np.save('df/main_test_data.npy', np.array(main_test_data))
print("Main_test_data saved with size(Bytes): {}".format(os.stat('df/main_test_data.npy').st_size))
np.save('df/values_test_data.npy', np.array(values_test_data))
print("Values_test_data saved with size(Bytes): {}".format(os.stat('df/values_test_data.npy').st_size))
np.save('df/order_test_data.npy', np.array(order_test_data))
print("Order_test_data saved with size(Bytes): {}".format(os.stat('df/order_test_data.npy').st_size))
print("Main bot test data length: {}".format(len(main_bot_data)))<jupyter_output>Main bot test data length: 683509<jupyter_text>Load data (if needed)<jupyter_code>main_data = np.load('df/main_data.npy').tolist()
values_data = np.load('df/values_data.npy').tolist()
order_data = np.load('df/order_data.npy').tolist()

main_df = pd.DataFrame(main_data)

main_df

list_device_class_code = pd.DataFrame(main_data).device_class_code.value_counts().index.tolist()
list_os_family_code = pd.DataFrame(main_data).os_family_code.value_counts().index.tolist()
list_os_code = pd.DataFrame(main_data).os_code.value_counts().index.tolist()
list_ua_class_code = pd.DataFrame(main_data).ua_class_code.value_counts().index.tolist()
list_ua_family_code = pd.DataFrame(main_data).ua_family_code.value_counts().index.tolist()
list_ua_version = pd.DataFrame(main_data).ua_version.value_counts().index.tolist()

print("Device count: {}".format(len(list_device_class_code)))
print("Device platform family count: {}".format(len(list_os_family_code)))
print("Device platform count: {}".format(len(list_os_code)))
print("Device browser class count: {}".format(len(list_ua_class_code)))
print("Device browser family count: {}".format(len(list_ua_family_code)))
print("Device browser version count: {}".format(len(list_ua_version)))
main_bot_data = np.load('df/main_bot_data.npy').tolist()
values_bot_data = np.load('df/values_bot_data.npy').tolist()
order_bot_data = np.load('df/order_bot_data.npy').tolist()

list_device_class_code = pd.DataFrame(main_bot_data)[1].value_counts().index.tolist()
list_os_family_code = pd.DataFrame(main_bot_data)[4].value_counts().index.tolist()
list_os_code = pd.DataFrame(main_bot_data)[3].value_counts().index.tolist()
list_ua_class_code = pd.DataFrame(main_bot_data)[6].value_counts().index.tolist()
list_ua_family_code = pd.DataFrame(main_bot_data)[7].value_counts().index.tolist()
list_ua_version = pd.DataFrame(main_bot_data)[8].value_counts().index.tolist()

print("Device count: {}".format(len(list_device_class_code)))
print("Device platform family count: {}".format(len(list_os_family_code)))
print("Device platform count: {}".format(len(list_os_code)))
print("Device browser class count: {}".format(len(list_ua_class_code)))
print("Device browser family count: {}".format(len(list_ua_family_code)))
print("Device browser version count: {}".format(len(list_ua_version)))
main_test_data = np.load('df/main_test_data.npy').tolist()
values_test_data = np.load('df/values_test_data.npy').tolist()
order_test_data = np.load('df/order_test_data.npy').tolist()

list_device_class_code = pd.DataFrame(main_test_data).device_class_code.value_counts().index.tolist()
list_os_family_code = pd.DataFrame(main_test_data).os_family_code.value_counts().index.tolist()
list_os_code = pd.DataFrame(main_test_data).os_code.value_counts().index.tolist()
list_ua_class_code = pd.DataFrame(main_test_data).ua_class_code.value_counts().index.tolist()
list_ua_family_code = pd.DataFrame(main_test_data).ua_family_code.value_counts().index.tolist()
list_ua_version = pd.DataFrame(main_test_data).ua_version.value_counts().index.tolist()

print("Device count: {}".format(len(list_device_class_code)))
print("Device platform family count: {}".format(len(list_os_family_code)))
print("Device platform count: {}".format(len(list_os_code)))
print("Device browser class count: {}".format(len(list_ua_class_code)))
print("Device browser family count: {}".format(len(list_ua_family_code)))
print("Device browser version count: {}".format(len(list_ua_version)))<jupyter_output>Device count: 6
Device platform family count: 27
Device platform count: 94
Device browser class count: 6
Device browser family count: 145
Device browser version count: 3172<jupyter_text>Train Part<jupyter_code>important_orders_keys_set = {
    'Upgrade-Insecure-Requests',
    'Accept', 
    'If-Modified-Since',
    'Host', 
    'Connection', 
    'User-Agent', 
    'From', 
    'Accept-Encoding' 
}

important_values_keys_set = {
    'Accept', 
    'Accept-Charset', 
    'Accept-Encoding'
}
from lib.parsers.logParser import LogParser

orders_vectorizer = sklearn.feature_extraction.DictVectorizer(sparse=True, dtype=float)
values_vectorizer = sklearn.feature_extraction.DictVectorizer(sparse=True, dtype=float)

l_parser = LogParser(log_folder='Logs/')
l_parser.reassign_orders_values(order_data, values_data)

full_sparce_dummy = l_parser.prepare_data(orders_vectorizer, values_vectorizer, important_orders_keys_set, important_values_keys_set, fit_dict=True)
from sklearn import preprocessing

lb = preprocessing.LabelBinarizer(sparse_output=True)

list_device_class_code = main_df.device_class_code.value_counts().index.tolist()
list_device_class_code.append('NaN')
lb.fit(list_device_class_code)

y_device_class = lb.transform(main_df.device_class_code.fillna('NaN'))
y_device_class.shape
from sklearn.model_selection import GridSearchCV, cross_val_predict, cross_val_score, train_test_split, KFold
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import roc_auc_score, roc_curve, f1_score, make_scorer
from sklearn.multiclass import OneVsRestClassifier
from sklearn import preprocessing

X_train, X_test, y_train, y_test = train_test_split(full_sparce_dummy, y_device_class, test_size=0.33, random_state=42)
%%time
clf = OneVsRestClassifier(LogisticRegression(random_state=42, C=100))
clf.fit(X_train, y_train)
from lib.thresholdPredictions import ThresholdPredictions

pred = ThresholdPredictions(user_agent_list=lb.classes_.tolist(), clf=clf)
y_test_names, y_predicted, compare_answers, answers_count = pred.predict(X_test, y_test, 0.13, sparce_y=True)
compare_frame = pd.concat(
    [
        y_test_names,
        y_predicted, 
        pd.DataFrame(compare_answers), 
        pd.DataFrame(answers_count)
    ], keys=['test', 'predicted', 'correctness', 'count'], axis=1, join='inner')

#print('Headers: ' + str(' '.join(important_keys)))
print('sparse_dummy_values.shape: ' + str(full_sparce_dummy.shape))
print("Answers Count (Average) = {0}".format(np.average(answers_count)))
smart_accuracy = numpy.sum(compare_answers) / y_test.shape[0]
print("True Positive = {0}".format(smart_accuracy))
compare_frame
pd.DataFrame(main_data).device_class_code
y_device_class[0].toarray()
list_device_class_code
lb.classes_.tolist()<jupyter_output><empty_output><jupyter_text>Load all data to full lists<jupyter_code>from lib.parsers.logParser import LogParser

l_parser = LogParser(log_folder='Logs/')

main_data, _, _ = l_parser.parse_train_sample(0, 128, filter_crawlers=True, parse_ua=True)
len(main_data)
main_df = pd.DataFrame(main_data)

main_df

list_device_class_code = main_df.device_class_code.value_counts().index.tolist()
list_os_family_code = main_df.os_family_code.value_counts().index.tolist()
list_os_code = main_df.os_code.value_counts().index.tolist()
list_ua_class_code = main_df.ua_class_code.value_counts().index.tolist()
list_ua_family_code = main_df.ua_family_code.value_counts().index.tolist()
list_ua_version = main_df.ua_version.value_counts().index.tolist()

print("Device count: {}".format(len(list_device_class_code)))
print("Device platform family count: {}".format(len(list_os_family_code)))
print("Device platform count: {}".format(len(list_os_code)))
print("Device browser class count: {}".format(len(list_ua_class_code)))
print("Device browser family count: {}".format(len(list_ua_family_code)))
print("Device browser version count: {}".format(len(list_ua_version)))
main_df.device_class_code.value_counts().hist()
plt.xlabel('Число значений')
plt.ylabel('Количество таких значений')
main_df.device_class_code.value_counts()
main_df.os_family_code.value_counts().hist()
plt.xlabel('Число значений')
plt.ylabel('Количество таких значений')
main_df.os_family_code.value_counts()
main_df.os_code.value_counts().hist()
plt.xlabel('Число значений')
plt.ylabel('Количество таких значений')
main_df.os_code.value_counts()
main_df.ua_class_code.value_counts().hist()
plt.xlabel('Число значений')
plt.ylabel('Количество таких значений')
main_df.ua_class_code.value_counts()
main_df.ua_family_code.value_counts().hist()
plt.xlabel('Число значений')
plt.ylabel('Количество таких значений')
main_df.ua_family_code.value_counts()
main_df.ua_family_code.value_counts()[main_df.ua_family_code.value_counts() > 100]
main_df.ua_version.value_counts().hist()
plt.xlabel('Число значений')
plt.ylabel('Количество таких значений')
main_df.ua_version.value_counts()
main_df.ua_family_code.value_counts()[main_df.ua_family_code.value_counts() > 100]<jupyter_output><empty_output><jupyter_text>62 from 5790 Browsers in TOP<jupyter_code>main_df.ua_family_code.value_counts()[main_df.ua_family_code.value_counts() < 100].hist()
main_df.ua_family_code.value_counts()[main_df.ua_family_code.value_counts() > 10]<jupyter_output><empty_output><jupyter_text>Only 117 browsers met over 10 timesSo we can safely assume that the error from the new unpopular browsers is minimal<jupyter_code>with open("df/group_labels/list_ua_version.txt", 'w') as f:
    for s in list_ua_version:
        f.write(s + '\n')

with open("df/group_labels/list_ua_family_code.txt", 'w') as f:
    for s in list_ua_family_code:
        f.write(s + '\n')

with open("df/group_labels/list_ua_class_code.txt", 'w') as f:
    for s in list_ua_class_code:
        f.write(s + '\n')
        
with open("df/group_labels/list_os_code.txt", 'w') as f:
    for s in list_os_code:
        f.write(s + '\n')

with open("df/group_labels/list_os_family_code.txt", 'w') as f:
    for s in list_os_family_code:
        f.write(s + '\n')

with open("df/group_labels/list_device_class_code.txt", 'w') as f:
    for s in list_device_class_code:
        f.write(s + '\n')<jupyter_output><empty_output><jupyter_text>Test Bot Data and other samples set<jupyter_code>from lib.parsers.logParser import LogParser

l_parser = LogParser(log_folder='Logs/')

main2_data, values2_data, order2_data = l_parser.parse_train_sample(30, 40, filter_crawlers=True, parse_ua=True)

np.save('df/main2_data.npy', np.array(main2_data))
print("Main2_data saved with size(Bytes): {}".format(os.stat('df/main2_data.npy').st_size))
np.save('df/values2_data.npy', np.array(values2_data))
print("Values2_data saved with size(Bytes): {}".format(os.stat('df/values2_data.npy').st_size))
np.save('df/order2_data.npy', np.array(order2_data))
print("Order2_data saved with size(Bytes): {}".format(os.stat('df/order2_data.npy').st_size))
from lib.parsers.logParser import LogParser

l_parser = LogParser(log_folder='Logs/')

main2_bot_data, values2_bot_data, order2_bot_data = l_parser.parse_bot_sample(50, 60, 40, 50, filter_crawlers=True, parse_ua=True)

np.save('df/main2_bot_data.npy', np.array(main2_bot_data))
print("Main2_bot_test_data saved with size(Bytes): {}".format(os.stat('df/main2_bot_data.npy').st_size))
np.save('df/values2_bot_data.npy', np.array(values2_bot_data))
print("Values2_bot_data saved with size(Bytes): {}".format(os.stat('df/values2_bot_data.npy').st_size))
np.save('df/order2_bot_data.npy', np.array(order2_bot_data))
print("Order2_bot_data saved with size(Bytes): {}".format(os.stat('df/order2_bot_data.npy').st_size))<jupyter_output>Main2_bot_test_data saved with size(Bytes): 34687220
Values2_bot_data saved with size(Bytes): 182712696
Order2_bot_data saved with size(Bytes): 27279318
{'fraction_non_alphanumeric': 0.13695813581586613, 'fraction_numerical': 0.013364555371523217, 'mean_word_length': 8.373661670235546, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 54, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '517466', 'license': 'MIT', 'n_tokens_mistral': 6573, 'n_tokens_neox': 6423, 'n_words': 954, 'path': '08-ParseUserAgent.ipynb', 'repo_name': 'levabd/http-ml-antifraud', 'size': '17509'}