<jupyter_start><jupyter_text>reference for calulating quartile [here](http://web.mnstate.edu/peil/MDEV102/U4/S36/S363.html:~:text=The%20third%20quartile%2C%20denoted%20by,25%25%20lie%20above%20Q3%20)<jupyter_code>mean(case_duration_dic.values())
# quartile calculation 
case_durations = list(case_duration_dic.values())
third_quartile  = calc_third_quartile(case_durations)
third_quartile
cases_gs = []
cases_gv = []
for k,v in case_duration_dic.items():
    if v <= third_quartile:
        cases_gs.append(k)
    else:
        cases_gv.append(k)
len(cases_gs), len(cases_gv)
gs_data = df.loc[df['CaseID'].isin(cases_gs)]
gv_data = df.loc[df['CaseID'].isin(cases_gv)]
tot = len(cases_gs)+ len(cases_gv)
percent_gs_cases = len(cases_gs) / tot
print(percent_gs_cases)
df.shape, len(cases_train), len(cases_val), len(cases_test)
data_train = df.loc[df['CaseID'].isin(cases_train)]
data_val = df.loc[df['CaseID'].isin(cases_val)]
data_test = df.loc[df['CaseID'].isin(cases_test)]
name = 'dataset/preprocessed/helpdesk_d2'
pickle.dump(data_train, open(name+"_train_RL.pkl", "wb"))
pickle.dump(data_val, open(name+"_val_RL.pkl", "wb"))
pickle.dump(data_test, open(name+"_test_RL.pkl", "wb"))
data_train
data_test
get_unique_act(data_train)
get_unique_act(data_test)
get_unique_act(data_val)
get_unique_act(df2)
get_unique_act(df)
get_unique_act(gs_data)
get_unique_act(gv_data)
print("stats after filtering dfg compliant cases")
plot_case_occ(df)
  
print("stats initial")
plot_case_occ(df2)<jupyter_output>stats initial
activity   occurance
1             4144
2             45
3             108
4             14
5             5
6             4150
7             4
8             4278
9             962
{'fraction_non_alphanumeric': 0.1334120425029516, 'fraction_numerical': 0.03955135773317592, 'mean_word_length': 4.865051903114187, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '290839', 'license': 'MIT', 'n_tokens_mistral': 706, 'n_tokens_neox': 657, 'n_words': 98, 'path': 'src/dataset_div/dataset_div_RL2.ipynb', 'repo_name': 'avani17101/goal-oriented-next-best-activity-recomendation', 'size': '1694'}