<commit_before>BEGIN {
   seen_count = 0;
}
{
   discard = 0;
   if ($1 ~ "[0-9]") {
	  seen[seen_count] = $1

	  for (i = seen_count-1; i >= 0; --i) {
		 file1 = destDir "/frames/Frame" $1 ".html"
		 file2 = destDir "/frames/Frame" seen[i] ".html"
		 diff_cmd = sprintf("diff -s --brief -I \"Arrival Time\" -I \"Frame [:digit:]*\" -I \"Resent Packet\" -I \"Suspected resend of frame\" %s %s > /dev/null", file1, file2);
		 if (system(diff_cmd) == 0) {
			report_dup = sprintf("echo '%s is identical to %s' 1>&2", file1, file2);
			system(report_dup);
			discard = 1;
			break;
		 }
	  }
	  seen_count++;
   }

   if (discard == 0) {
	  print $0;
   }
}<commit_msg>-Added some ignore cases in remove dups awk scripts<commit_after>BEGIN {
   seen_count = 0;
}
{
   discard = 0;
   if ($1 ~ "[0-9]") {
	  seen[seen_count] = $1

	  for (i = seen_count-1; i >= 0; --i) {
		 file1 = destDir "/frames/Frame" $1 ".html"
		 file2 = destDir "/frames/Frame" seen[i] ".html"
		 diff_cmd = sprintf("diff -s --brief -I \"Arrival Time\" -I \"Frame [:digit:]*\" -I \"Resent Packet\" -I \"Suspected resend of frame\" -I \"Request Frame: [:digit:]*\" -I \"Response Time (ms): [:digit:]*\" -I \"Release Time (ms): [:digit:]*\" %s %s > /dev/null", file1, file2);
		 #diff_cmd = sprintf("diff -s --brief -I \"Arrival Time\" -I \"Frame [:digit:]*\" -I \"Resent Packet\" -I \"Suspected resend of frame\" %s %s > /dev/null", file1, file2);
		 if (system(diff_cmd) == 0) {
			report_dup = sprintf("echo '%s is identical to %s' 1>&2", file1, file2);
			system(report_dup);
			discard = 1;
			break;
		 }
	  }
	  seen_count++;
   }

   if (discard == 0) {
	  print $0;
   }
}
{'fraction_non_alphanumeric': 0.21467556094602788, 'fraction_numerical': 0.026682838083687082, 'mean_word_length': 2.6666666666666665, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '892632', 'n_tokens_mistral': 723, 'n_tokens_neox': 642, 'n_words': 189}