<jupyter_start><jupyter_text>Dependencies and Setup  Install Process Mining for Python<jupyter_code>!pip install pm4py
!pip install jellyfish

from pprint import pprint
import pm4py
import copy
import numpy
import jellyfish
import math<jupyter_output><empty_output><jupyter_text>Import dataset and save as event log<jupyter_code>from google.colab import drive
drive.mount('/content/drive')
log_file = '/content/drive/MyDrive/pathToXesLogFile'
log = pm4py.read_xes(log_file)<jupyter_output>Mounted at /content/drive<jupyter_text>Preprocessing  Search for string attributes and save the according max string lenght for every attribute<jupyter_code># All string keys

seen_keys = []

for t in log:
    for i in range(len(t)):
        event = t[i]
        keys = event.keys()
        for key in keys:
            if (type(event[key]) == str):
                seen_keys.append(key)
                seen_keys = list(dict.fromkeys(seen_keys))

string_keys = dict.fromkeys(seen_keys)

keys2 = string_keys.keys()
for key in keys2:
    string_keys[key] = 0

#max string_keys
for t in log:
    for i in range(len(t)):
        for key in keys2:
            if key in t[i]:
                if len(t[i][key]) > string_keys[key]:
                    string_keys[key] = len(t[i][key])
for t in log:
  for i in range(len(t)):
    event = t[i]
    keys = event.keys()
    for key in keys:
      if (type(event[key]) == str):
        seen_keys.append(key)
        seen_keys = list(dict.fromkeys(seen_keys))<jupyter_output><empty_output><jupyter_text>Search for all numeric attributes and save the attribute names<jupyter_code># All numeric keys

seen_keys = []

for t in log:
  for i in range(len(t)):
    event = t[i]
    keys = event.keys()
    for key in keys:
      if (type(event[key]) == float or type(event[key]) == int):
        seen_keys.append(key)
        seen_keys = list(dict.fromkeys(seen_keys))

seen_keys = dict.fromkeys(seen_keys)<jupyter_output><empty_output><jupyter_text>Calculate the mean values of all numeric attributes<jupyter_code># mean values
sum = 0

seen_keys = []

for t in log:
  for i in range(len(t)):
    event = t[i]
    keys = event.keys()
    for key in keys:
      if (type(event[key]) == float or type(event[key]) == int):
        seen_keys.append(key)
        seen_keys = list(dict.fromkeys(seen_keys))

mean_keys = dict.fromkeys(seen_keys)
keys2 = mean_keys.keys()
for key in keys2:
  mean_keys[key] = 0

counter_keys = copy.deepcopy(mean_keys)

for t in log:
  for i in range(len(t)):
    for key in keys2:
      if key in t[i]  :
        mean_keys[key] += t[i][key]
        counter_keys[key] += 1

for key in keys2:
  mean_keys[key] = mean_keys[key] / counter_keys[key]
  print(mean_keys[key])<jupyter_output>63.6816560395868
17.815603519745885
113.2466582430006
0.0788787657112456
11.948984392276309
46.41002731923214
0.0<jupyter_text>Calculate the std. deviation of all numeric values<jupyter_code># std deviation of values

deviation_keys = dict.fromkeys(seen_keys)
keys2 = deviation_keys.keys()
for key in keys2:
  deviation_keys[key] = []

for t in log:
  for i in range(len(t)):
    for key in keys2:
      if key in t[i]:
        deviation_keys[key].append(t[i][key])

for key in keys2:
  deviation_keys[key] = numpy.std(deviation_keys[key])<jupyter_output>89.67779636777881
36.77930126804085
68.21924855798667
0.5776847046687658
4.023171014461607
39.345307223637775
0.0<jupyter_text>Apply the z score normalisation to the log<jupyter_code># z score of values

numeric_keys = dict.fromkeys(seen_keys)

z_log = copy.deepcopy(log)

for t in z_log:
  for i in range(len(t)):
    for key in numeric_keys.keys():
      if key in t[i] and deviation_keys[key] != 0:
        t[i][key] = (t[i][key] - mean_keys[key] ) / deviation_keys[key]<jupyter_output><empty_output><jupyter_text>Calculate the minimum and maximum timestamp per trace<jupyter_code># timestamp min max values per trace

min_timestamps = []

max_timestamps = []

for t in log:
  temp_min_timestamp = t[0]['time:timestamp']
  temp_max_timestamp = t[0]['time:timestamp']
  for i in range(len(t)):
    if temp_min_timestamp > t[i]['time:timestamp']:
      temp_min_timestamp = t[i]['time:timestamp']
    if temp_max_timestamp < t[i]['time:timestamp']:
      temp_max_timestamp = t[i]['time:timestamp']
  min_timestamps.append(temp_min_timestamp)
  max_timestamps.append(temp_max_timestamp)<jupyter_output><empty_output><jupyter_text>Apply min-max normalisation on timestamps<jupyter_code># normalize timestamps

normalised_log = copy.deepcopy(z_log)

for y in range (len(log)):
  for i in range (len(log[y])):
    if max_timestamps[y] == min_timestamps[y]:
      normalised_time = 0
    else: 
      normalised_time = (log[y][i]['time:timestamp'] - min_timestamps[y]) / (max_timestamps[y] - min_timestamps[y])
    normalised_log[y][i]['time:timestamp'] = normalised_time<jupyter_output><empty_output><jupyter_text>Define distance algorithm<jupyter_code># distance algorithm

def similarity(dict1, dict2, treshhold, weights):
    longest_value = 0
    distance_score = 0
    distance_sum = 0
    d1_keys = (dict1.keys())
    d2_keys = (dict2.keys())

    # Combine the keys from both events to calculate the vector distance between both of them
    both_keys = list(dict.fromkeys(dict1))
    for key in d2_keys:
      if not (key in both_keys):
        both_keys.append(key)

    both_keys = dict.fromkeys(both_keys)
    for key in both_keys:
      ## Timestamp distance
      if (key == 'time:timestamp'):
        diff = (dict1[key] - dict2[key])
        both_keys[key] = pow(diff, 2)

      # If no value is present for the key set a default value, otherwise take value from event

      # Check if value is numeric
      elif key in numeric_keys:
        if key not in dict1:
          d1_score = 0 # is 0 the best solution here?
        else:
          d1_score = dict1[key] 
        if key not in dict2:
          d2_score = 0 # is 0 the best solution here?
        else:
          d2_score = dict2[key]

        diff = d1_score - d2_score
        both_keys[key] = pow(diff, 2)
        if(key == 'totalPaymentAmount'):
          both_keys[key] = both_keys[key]


      # Otherwise value must be a string
      else:
        if key not in dict1:
          d1_string = '' # is '' the best solution here?
        else:
          d1_string = dict1[key] 
        if key not in dict2:
          d2_string = '' # is '' the best solution here?
        else:
          d2_string = dict2[key]

        # calculate levenstein distance
        lev_dist = jellyfish.levenshtein_distance(d1_string, d2_string)
        # execute normalization
        distance_percentage = lev_dist / string_keys[key]
        both_keys[key] = pow(distance_percentage, 2)
     
    # include weights and calculate euclidean distance
    for key in both_keys:
      distance_sum +=  weights[key] * both_keys[key]
    
    distance_score = math.sqrt(distance_sum) 

    # If distanz is under treshhold return true
    if distance_score < treshhold:
      #print(distance_score)
      return True
    else:
      return False<jupyter_output><empty_output><jupyter_text>Create weight dictionary<jupyter_code>seen_keys = []

for t in log:
  for i in range(len(t)):
    event = t[i]
    keys = event.keys()
    for key in keys:
        seen_keys.append(key)
        seen_keys = list(dict.fromkeys(seen_keys))

weights = dict.fromkeys(seen_keys)
for key in seen_keys:
    weights[key] = 1<jupyter_output>['amount', 'org:resource', 'dismissal', 'concept:name', 'vehicleClass', 'totalPaymentAmount', 'lifecycle:transition', 'time:timestamp', 'article', 'points', 'expense', 'notificationType', 'lastSent', 'paymentAmount', 'matricola']<jupyter_text>Example execution with weights and treshhold of 0.1<jupyter_code>weights1 = copy.deepcopy(weights)
#f.i. weights1['attribute']=10

treshhold = 0.1

for x in range(len(normalised_log)):
  t = normalised_log[x]
  for i in range(len(t)):
    event_outer = t[i]
    for y in range(i+1, len(t)):
      event_inner = t[y]
      flag = similarity(event_outer, event_inner, treshhold, weights1)
      if flag == True:
        print(t)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11002085122040967, 'fraction_numerical': 0.03348460689316816, 'mean_word_length': 3.4950385887541344, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '584383', 'license': 'MIT', 'n_tokens_mistral': 3005, 'n_tokens_neox': 2818, 'n_words': 712, 'path': 'Final_Submission.ipynb', 'repo_name': 'Jostafarr/duplicate-detection-algorithm', 'size': '8153'}