<jupyter_start><jupyter_text>FWAF Malicious Queries Datasethttps://github.com/faizann24/Fwaf-Machine-Learning-driven-Web-Application-Firewall<jupyter_code># Dataset paths
dataset_base_path = r"D:\\dev\\CyberML\\Datasets\\FWAF"

queries_raw = []
for fn in glob.glob(os.path.join(dataset_base_path, "*.txt")):
    queries_raw.append(os.path.normpath(fn))

# Make sure files get added in specific order
queries_raw.sort(reverse=True)
mapping_label = {0:"good", 1:"bad"}

print(queries_raw)
print(mapping_label)
def ends(df, x=5):
    '''Get the head and tail of a dataframe'''
    return df.head(x).append(df.tail(x))

raw_urls = []
for idx, fn in enumerate(queries_raw):
    with open(fn,'r', encoding='utf-8') as f:
        data = f.readlines()
        data = [(str(urllib.parse.unquote(d)), idx) for d in data]
        raw_urls.extend(data)

# Create dataframe with columns for URL and label
urldata = pd.DataFrame(raw_urls, columns=['url', 'label'])
urldata['url'] = urldata['url'].apply(lambda x: x.strip("\n"))
# Convert gt column according to mapping
# urldata['label'] = urldata['label'].apply(lambda x: mapping_label[x])
only_once_flag = 1

# View Dataset information and what the samples look like
print(urldata.info())
display(urldata.groupby('label').count())
display(ends(urldata))
# Create new columns to count specific features in each sample and use them for our ML classifier
if only_once_flag:
    urldata['url_length'] = urldata['url'].apply(lambda i: len(str(i)))
    urldata['count-'] = urldata['url'].apply(lambda i: i.count('-'))
    urldata['count@'] = urldata['url'].apply(lambda i: i.count('@'))
    urldata['count?'] = urldata['url'].apply(lambda i: i.count('?'))
    urldata['count%'] = urldata['url'].apply(lambda i: i.count('%'))
    urldata['count.'] = urldata['url'].apply(lambda i: i.count('.'))
    urldata['count='] = urldata['url'].apply(lambda i: i.count('='))
    urldata['count<'] = urldata['url'].apply(lambda i: i.count('<'))
    urldata['count>'] = urldata['url'].apply(lambda i: i.count('>'))
    urldata['count/'] = urldata['url'].apply(lambda i: i.count('/'))
    urldata['count\\'] = urldata['url'].apply(lambda i: i.count('\\'))
    urldata['count<'] = urldata['url'].apply(lambda i: 1 if i.count('<') > 3 else 0)

    # Count number of non alphanumeric characters in url
    urldata["count_non_alphanum"] = urldata.apply(lambda p: sum( not q.isalnum() for q in p["url"] ), axis=1)
    
    only_once_flag = 0
display(ends(urldata))
# column_to_display = 'count_non_alphanum'
# column_to_display = 'count@'
# grouped_data = urldata.groupby(['label',]).agg({column_to_display: 'sum'})
# grouped_data["%"] = grouped_data.apply(lambda x:  100*x / x.sum())
# grouped_data = grouped_data.reset_index()
# grouped_data

# Display total sum of each column by class
grouped_data = urldata.groupby(['label',]).agg('sum')
display("Sum", grouped_data)
# Display percentages of total
display("Percentage", grouped_data.apply(lambda x:  100*x / x.sum()))
# Display average number
display("Mean", urldata.groupby(['label',]).agg('mean'))
# display(urldata.groupby(['label',]).agg(['mean', 'sum']))
# Correlation Heatmap
corrmat = urldata.corr()
f, ax = plt.subplots(figsize=(25,19))
sns.heatmap(corrmat, square=True, annot = True, annot_kws={'size':10}, cmap='Blues')

plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18933493109646496, 'fraction_numerical': 0.007189934092270821, 'mean_word_length': 5.372137404580153, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 8, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '133616', 'license': 'MIT', 'n_tokens_mistral': 1234, 'n_tokens_neox': 1203, 'n_words': 254, 'path': 'notebooks/data_exploration_fwaf.ipynb', 'repo_name': 'enzo-ca/multilabel-url-classification', 'size': '3338'}