<jupyter_start><jupyter_text>Table of Contents1&nbsp;&nbsp;PDC 2020 and 20212&nbsp;&nbsp;BINET Logs  Dataset Descriptions:<jupyter_code>%load_ext autoreload
%autoreload 2
%matplotlib inline<jupyter_output><empty_output><jupyter_text>PDC 2020 and 2021<jupyter_code>#export
from dapnn.imports import *
from dapnn.data_processing import *

from dapnn.anomaly import *
def get_ds_stats(path):
    res=[]
    for log in progress_bar(glob.glob(path)):
        splits = log.split('.')[0].split('_')
        log_name = splits[-1]
        log = import_log(log)
        number_cases = len(log.index.unique())
        number_activities= len(log.activity.unique())
        number_events = len(log)
        number_attr = 1
        number_anomalies= len(log[log['normal']==False]['trace_id'].unique()) if 'normal' in log.columns else 'unknown'
        res.append([log_name,number_cases,number_activities,number_events
                    ,number_anomalies])
    res= pd.DataFrame(res,columns=['Name','# Cases','# Activities','# Events','# Anomalies'])
    return res
pdc20_train= get_ds_stats('data/csv/PDC2020_training/*')
pdc20_train
pdc20_gt= get_ds_stats('data/csv/PDC2020_ground_truth/*')
pdc20_gt
def format_ds(x):
    min = x.min()   
    max = x.max()
    return  min if min == max else f'{x.min()}-{x.max()}'

pdc20_train_stats = pdc20_train.apply(format_ds).to_frame().T
pdc20_gt_stats = pdc20_gt.apply(format_ds).to_frame().T

pdc21_train= get_ds_stats('data/csv/PDC2021_training/*')
pdc21_train
pdc21_gt= get_ds_stats('data/csv/PDC2021_ground_truth/*')
pdc21_gt
pdc21_train_stats = pdc21_train.apply(format_ds).to_frame().T
pdc21_gt_stats = pdc21_gt.apply(format_ds).to_frame().T

res =pd.concat([pdc20_train_stats,pdc20_gt_stats,pdc21_train_stats,pdc21_gt_stats])
res =res[res.columns[1:]]
res.index = ['PDC 2020 Train', 'PDC 2020 Test','PDC 2021 Train','PDC 2021 Test']
res['# Logs'] = [len(pdc20_train),len(pdc20_gt),len(pdc21_train),len(pdc21_gt)]
res =res[['# Logs','# Cases','# Activities','# Events','# Anomalies']]
res

print(res.to_latex())<jupyter_output>\begin{tabular}{lrllll}
\toprule
{} &  \# Logs & \# Cases & \# Activities &    \# Events & \# Anomalies \\
\midrule
PDC 2020 Train &     192 &    1000 &        16-38 &  8867-70106 &     unknown \\
PDC 2020 Test  &     192 &    1000 &        16-38 &  8764-68706 &     412-515 \\
PDC 2021 Train &     480 &    1000 &        37-65 &  9867-32009 &     unknown \\
PDC 2021 Test  &      96 &     250 &        35-64 &  6612-11860 &         125 \\
\bottomrule
\end{tabular}<jupyter_text>BINET Logs<jupyter_code>res=[]
for fn in progress_bar(glob.glob('data/csv/binet_logs/*')):
    log_name = fn.split('/')[-1][:-7]
    log = import_log(fn)
    number_cases = len(log.index.unique())
    number_activities= len(log.activity.unique())
    number_events = len(log)
    number_attr =len(get_attr(attr_dict,fn))-1
    number_anomalies= len(log[log['anomaly']!='normal']['trace_id'].unique())
    res.append([log_name,number_cases,number_activities,number_events,number_attr,number_anomalies])
binet_logs= pd.DataFrame(res,columns=['Name','# Cases','# Activities','# Events','# Attributes','# Anomalies'])
binet_logs
groups = np.unique(binet_logs['Name'].apply(lambda x: x.split('-')[0]).values)
dfs =[]
for group in groups:
    df = binet_logs[binet_logs.Name.str.contains(group)]
    len_df=len(df)
    df = df.apply(format_ds).to_frame().T
    df['# Logs'] = [len_df]
    dfs.append(df)
res =pd.concat(dfs)
res =res[res.columns[1:]]
res.index=[i.upper() if 'bpi' in i else i.capitalize() for i in groups ]
res = res[['# Logs','# Cases','# Activities','# Events','# Attributes','# Anomalies']].sort_index()
res
print(res.to_latex())<jupyter_output>\begin{tabular}{lrlllll}
\toprule
{} &  \# Logs &      \# Cases & \# Activities &        \# Events & \# Attributes & \# Anomalies \\
\midrule
BPIC12   &       1 &        13087 &           73 &          289892 &            0 &        3927 \\
BPIC13   &       3 &     819-7554 &        11-27 &      4068-81524 &            7 &    162-2257 \\
BPIC15   &       5 &     832-1409 &      417-491 &     46110-62667 &            6 &     232-438 \\
BPIC17   &       2 &  31509-42995 &        17-53 &  285211-1269176 &            2 &  9398-13193 \\
Gigantic &       4 &         5000 &      152-157 &     38774-42711 &          1-4 &   1499-1553 \\
Huge     &       4 &         5000 &          109 &     46919-53627 &          1-4 &   1416-1479 \\
Large    &       4 &         5000 &           85 &     61789-67524 &          1-4 &   1482-1529 \\
Medium   &       4 &         5000 &           65 &     38990-41991 &          1-4 &   1459-1550 \\
P2p      &       4 &         5000 &           27 &     48477-53193[...]
{'fraction_non_alphanumeric': 0.1589798087141339, 'fraction_numerical': 0.09925611052072264, 'mean_word_length': 2.782958199356913, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '488888', 'license': 'Apache-2.0', 'n_tokens_mistral': 2063, 'n_tokens_neox': 1747, 'n_words': 306, 'path': '07_dataset_statistics.ipynb', 'repo_name': 'joLahann/dapnn', 'size': '4705'}