<jupyter_start><jupyter_text>Normalize text without Language ModelWe are using string similarity (Normalized Levenshtein score) to replace out-of-vocabulary words.<jupyter_code>from strsimpy.normalized_levenshtein import NormalizedLevenshtein
from nltk.corpus import words
import string
import nltk
nltk.download('punkt')
nltk.download('words')

normalized_levenshtein = NormalizedLevenshtein()

def levensthein_score(word1, word2):
  return normalized_levenshtein.similarity(word1, word2)
vocab = [x.lower() for x in words.words()]
len(vocab    )
from nltk.tokenize import word_tokenize

def normalize_data(text):
  words = word_tokenize(text)
  
  normalized_text=''
  for word in words:
    if word in string.punctuation:
      normalized_text = normalized_text+word
    elif word not in vocab:
      word=word.lower()
      max_score=0
      replace_word=word
      for v in vocab:
        score = levensthein_score(word, v)
        if score>max_score:
          max_score=score
          replace_word=v
      normalized_text = normalized_text + ' ' + replace_word
    else:
      normalized_text = normalized_text + ' ' + word
  return normalized_text.strip()<jupyter_output><empty_output><jupyter_text>Evaluation on Test Data<jupyter_code>import pandas as pd
import os

pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', None)
pd.set_option('display.max_colwidth', None)
path = 'drive/MyDrive/CS685'

input_df = pd.read_csv(os.path.join(path, "xsum_test_baseline.csv"),sep='\t')
input_df.head(2)
test_df = input_df
test_df['normalized_text'] = test_df['text'].apply(normalize_data)
test_df.to_csv(os.path.join(path, "xsum_test_pred_baseline.csv"), index=False, sep='\t')
test_df.head(5)
test_df = pd.read_csv(os.path.join(path, "xsum_test_pred_baseline.csv"),sep='\t')
import difflib
def get_dissimilar_spans(orig_words, gt_words, pred_words):
  gt_matcher = difflib.SequenceMatcher(a=orig_words, b=gt_words)
  pred_matcher = difflib.SequenceMatcher(a=gt_words, b=pred_words)
  orig_spans = []
  gt_spans = []
  pred_spans = []
  mismatch_spans = []
  for codes in gt_matcher.get_opcodes():
    op,a_start,a_end,b_start,b_end = codes
    if op == 'replace':
      orig_spans.append(" ".join(orig_words[a_start:a_end]))
      gt_spans.append(" ".join(gt_words[b_start:b_end]))

  for codes in pred_matcher.get_opcodes():
    op,a_start,a_end,b_start,b_end = codes
    if op == 'replace':
      pred_spans.append(" ".join(pred_words[b_start:b_end]))
      mismatch_spans.append(" ".join(gt_words[a_start:a_end]))
  
  return orig_spans, gt_spans, pred_spans, mismatch_spans

def get_stats_for_predictions(orig_text, gt_text, pred_text):
  orig_words = nltk.word_tokenize(orig_text)
  gt_words = nltk.word_tokenize(gt_text)
  pred_words = nltk.word_tokenize(pred_text)
  orig_words = [word.lower().strip() for word in orig_words]
  gt_words = [word.lower().strip() for word in gt_words]
  pred_words = [word.lower().strip() for word in pred_words]
  correct_preds = []
  wrong_preds = []
  changed_orig_words = []
  changed_gt_words = []
  replaced_word_cnt = 0
  correct_pred_cnt = 0
  if len(orig_words)!= len(gt_words):
    print(orig_text)
    print(gt_text)
  elif len(gt_words)!=len(pred_words):
    orig_spans, gt_spans, pred_spans, mismatch_spans = get_dissimilar_spans(orig_words, gt_words, pred_words)
    wrong_preds = pred_spans
    changed_orig_words = orig_spans
    changed_gt_words = gt_spans
    replaced_word_cnt = len(gt_spans)
    correct_pred_cnt = len(gt_spans) - len(mismatch_spans)
    correct_preds = list(set(gt_spans)-set(mismatch_spans))
  else:
    for i in range(len(orig_words)):
      orig_word = orig_words[i]
      gt_word = gt_words[i]
      pred_word = pred_words[i]
      if orig_word != gt_word:
        changed_orig_words.append(orig_word)
        changed_gt_words.append(gt_word)
        replaced_word_cnt = replaced_word_cnt+1
        if pred_word == gt_word:
          correct_preds.append(pred_word)
          correct_pred_cnt = correct_pred_cnt+1
        else:
          wrong_preds.append(pred_word)

  return {"replaced_gt_words":changed_gt_words,
          "replaced_original_words": changed_orig_words,
          "replaced_word_count": replaced_word_cnt,
          "correct_predictions": correct_preds,
          "correct_prediction_count": correct_pred_cnt,
          "wrong_predictions": wrong_preds}

def get_accuracy_df(input_df, pred_df):
  pred_df = pred_df.drop(columns=['gt_text', 'text'])
  df = pd.concat([input_df, pred_df], axis=1)
  df["Stats"] = df.apply(lambda x: get_stats_for_predictions(x["text"], x["gt_text"],x["normalized_text"]), axis = 1)
  df = pd.concat([df.drop(['Stats'], axis=1), df['Stats'].apply(pd.Series)], axis=1)
  return df
stat_df = get_accuracy_df(input_df,test_df)
stat_df.head(5)
print(f"Total incorrect tokens: {stat_df['replaced_word_count'].sum()}\n Total correct predictions: {stat_df['correct_prediction_count'].sum()} \nTest accuracy: {stat_df['correct_prediction_count'].sum()/stat_df['replaced_word_count'].sum()}")<jupyter_output>Total incorrect tokens: 232
 Total correct predictions: 72 
Test accuracy: 0.3103448275862069
{'fraction_non_alphanumeric': 0.11713286713286714, 'fraction_numerical': 0.007964257964257964, 'mean_word_length': 4.864464692482915, 'pattern_counts': {'":': 6, '<': 8, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '280206', 'license': 'MIT', 'n_tokens_mistral': 1943, 'n_tokens_neox': 1898, 'n_words': 319, 'path': 'Baseline_Model.ipynb', 'repo_name': 'nalsingh26/685_Normalize_Text', 'size': '5148'}