<commit_before>import sys
import pickle as pkl

def load_constants():
  CONSTANTS_PATH = 'constants'

  with open(CONSTANTS_PATH + '/ARABIC_LETTERS_LIST.pickle', 'rb') as file:
    ARABIC_LETTERS_LIST = pkl.load(file)

  with open(CONSTANTS_PATH + '/CLASSES_LIST.pickle', 'rb') as file:
    CLASSES_LIST = pkl.load(file)

  return ARABIC_LETTERS_LIST, CLASSES_LIST

def get_diacritic_class(idx, line, case_ending, ARABIC_LETTERS_LIST, CLASSES_LIST):
  idxs = [idx + 1, idx + 2, idx + 3]

  # Handle without case ending
  if not case_ending:
    end = True

    if idxs[0] < len(line) and line[idxs[0]] in ARABIC_LETTERS_LIST:
      end = False
    elif idxs[0] < len(line) and idxs[1] < len(line) and \
         line[idxs[0]] in CLASSES_LIST and line[idxs[1]] in ARABIC_LETTERS_LIST:
      end = False
    elif idxs[0] < len(line) and idxs[1] < len(line) and idxs[2] < len(line) and \
         line[idxs[0]] in CLASSES_LIST and line[idxs[1]] in CLASSES_LIST and \
         line[idxs[2]] in ARABIC_LETTERS_LIST:
      end = False

    if end:
      return -1

  if idxs[0] >= len(line) or line[idxs[0]] not in CLASSES_LIST:
    # No diacritic
    return 0

  diac = line[idx + 1]

  if idxs[1] >= len(line) or line[idxs[1]] not in CLASSES_LIST:
    # Only one diacritic
    return CLASSES_LIST.index(diac) + 1

  diac += line[idxs[1]]

  try:
    # Try the possibility of double diacritics
    return CLASSES_LIST.index(diac) + 1
  except:
    try:
      # Try the possibility of reversed diacritics
      return CLASSES_LIST.index(diac[::-1]) + 1
    except:
      # Otherwise consider only the first diacritic
      return CLASSES_LIST.index(diac[0]) + 1

def get_diacritics_classes(line, case_ending, ARABIC_LETTERS_LIST, CLASSES_LIST):
  classes = list()
  for idx, char in enumerate(line):
    if char in ARABIC_LETTERS_LIST:
      classes.append(get_diacritic_class(idx, line, case_ending, ARABIC_LETTERS_LIST, CLASSES_LIST))
  return classes

def calculate_der(original_file, output_file, case_ending=True, no_diacritic=True):
  ARABIC_LETTERS_LIST, CLASSES_LIST = load_constants()

  with open(original_file, 'r') as file:
    original_content = file.readlines()

  with open(output_file, 'r') as file:
    output_content = file.readlines()

  assert(len(original_content) == len(output_content))
<commit_msg>fix a minor bug in der_wer_calculator.py<commit_after>import sys
import pickle as pkl

def load_constants():
  CONSTANTS_PATH = 'constants'

  with open(CONSTANTS_PATH + '/ARABIC_LETTERS_LIST.pickle', 'rb') as file:
    ARABIC_LETTERS_LIST = pkl.load(file)

  with open(CONSTANTS_PATH + '/CLASSES_LIST.pickle', 'rb') as file:
    CLASSES_LIST = pkl.load(file)

  return ARABIC_LETTERS_LIST, CLASSES_LIST

def get_diacritic_class(idx, line, case_ending, ARABIC_LETTERS_LIST, CLASSES_LIST):

  # Handle without case ending
  if not case_ending:
    end = True
    for i in range(idx + 1, len(line)):
      if line[i] not in CLASSES_LIST:
        end = line[i] not in ARABIC_LETTERS_LIST
        break
    if end:
      return -1

  if idx + 1 >= len(line) or line[idx + 1] not in CLASSES_LIST:
    # No diacritic
    return 0

  diac = line[idx + 1]

  if idx + 2 >= len(line) or line[idx + 2] not in CLASSES_LIST:
    # Only one diacritic
    return CLASSES_LIST.index(diac) + 1

  diac += line[idx + 2]

  try:
    # Try the possibility of double diacritics
    return CLASSES_LIST.index(diac) + 1
  except:
    try:
      # Try the possibility of reversed double diacritics
      return CLASSES_LIST.index(diac[::-1]) + 1
    except:
      # Otherwise consider only the first diacritic
      return CLASSES_LIST.index(diac[0]) + 1

def get_diacritics_classes(line, case_ending, ARABIC_LETTERS_LIST, CLASSES_LIST):
  classes = list()
  for idx, char in enumerate(line):
    if char in ARABIC_LETTERS_LIST:
      classes.append(get_diacritic_class(idx, line, case_ending, ARABIC_LETTERS_LIST, CLASSES_LIST))
  return classes

def calculate_der(original_file, output_file, case_ending=True, no_diacritic=True):
  ARABIC_LETTERS_LIST, CLASSES_LIST = load_constants()

  with open(original_file, 'r') as file:
    original_content = file.readlines()

  with open(output_file, 'r') as file:
    output_content = file.readlines()

  assert(len(original_content) == len(output_content))

{'fraction_non_alphanumeric': 0.10195530726256984, 'fraction_numerical': 0.010242085661080074, 'mean_word_length': 3.806487695749441, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6861319', 'n_tokens_mistral': 1688, 'n_tokens_neox': 1567, 'n_words': 397}