<commit_before>import numpy as np
import sys, argparse

args = argparse.ArgumentParser \
  ( description = "Compares FILE1 and FILE2 numerically \
      by calculating L1 norms column-wise between the two input files. \
      FILE1 and FILE2 must have the same number of values. FILE1 and FILE2 \
      are the same if the norm of every column is below TOL.", \
    epilog = "Exit status is 0 if files are the same, 1 if different, \
      2 if trouble." )
      
args.add_argument ( "FILE1", metavar = "FILE1", action = "store" )
args.add_argument ( "FILE2", metavar = "FILE2", action = "store" )

args.add_argument \
  ( "--skip", dest = "skip", action = "store", \
    help = "Skip reading the first SKIP lines." )
args.add_argument \
  ( "--prec", dest = "prec", action = "store", \
    help = "Precision to use, either 'single' or 'double' (default)." )
args.add_argument \
  ( "--tol", dest = "tol", action = "store", \
    help = "The norm above which the values are considered different. \
            Default is 10 times machine epsilon for PREC." )

args = args.parse_args()

FileStart = args.FILE1 
FileEnd   = args.FILE2

Skip = 0
if ( args.skip ) : Skip = int ( args.skip )

Precision = 'double'
if ( args.prec) : Precision = args.prec

Tolerance = np.finfo(Precision).eps * 10.0
if ( args.tol ) : Tolerance = float ( args.tol )

try: 

  Start = np.loadtxt ( FileStart, dtype = Precision, skiprows = Skip )
  End   = np.loadtxt ( FileEnd, dtype = Precision, skiprows = Skip )

  DistanceSum = np.abs(End - Start).sum(axis = 0)
  OriginSum = np.abs(Start).sum(axis=0)
  print("OriginSum:", OriginSum)
  print("DistanceSum:", DistanceSum)
  L1_Error = np.where ( OriginSum > 0.0, DistanceSum / OriginSum, DistanceSum )

  if ( np.all ( L1_Error <= Tolerance ) ):
    sys.exit(0)
  else:
    print ( "Tolerance : %E" % Tolerance )
    print ( "L1 norms  :", L1_Error )
    sys.exit(1)

except Exception as e:
  print(e);
  sys.exit(2)<commit_msg>Removed debugging print in numdiff.py.<commit_after>import numpy as np
import sys, argparse

args = argparse.ArgumentParser \
  ( description = "Compares FILE1 and FILE2 numerically \
      by calculating L1 norms column-wise between the two input files. \
      FILE1 and FILE2 must have the same number of values. FILE1 and FILE2 \
      are the same if the norm of every column is below TOL.", \
    epilog = "Exit status is 0 if files are the same, 1 if different, \
      2 if trouble." )
      
args.add_argument ( "FILE1", metavar = "FILE1", action = "store" )
args.add_argument ( "FILE2", metavar = "FILE2", action = "store" )

args.add_argument \
  ( "--skip", dest = "skip", action = "store", \
    help = "Skip reading the first SKIP lines." )
args.add_argument \
  ( "--prec", dest = "prec", action = "store", \
    help = "Precision to use, either 'single' or 'double' (default)." )
args.add_argument \
  ( "--tol", dest = "tol", action = "store", \
    help = "The norm above which the values are considered different. \
            Default is 10 times machine epsilon for PREC." )

args = args.parse_args()

FileStart = args.FILE1 
FileEnd   = args.FILE2

Skip = 0
if ( args.skip ) : Skip = int ( args.skip )

Precision = 'double'
if ( args.prec) : Precision = args.prec

Tolerance = np.finfo(Precision).eps * 10.0
if ( args.tol ) : Tolerance = float ( args.tol )

try: 

  Start = np.loadtxt ( FileStart, dtype = Precision, skiprows = Skip )
  End   = np.loadtxt ( FileEnd, dtype = Precision, skiprows = Skip )

  DistanceSum = np.abs(End - Start).sum(axis = 0)
  OriginSum = np.abs(Start).sum(axis=0)
  L1_Error = np.where ( OriginSum > 0.0, DistanceSum / OriginSum, DistanceSum )

  if ( np.all ( L1_Error <= Tolerance ) ):
    sys.exit(0)
  else:
    print ( "Tolerance : %E" % Tolerance )
    print ( "L1 norms  :", L1_Error )
    sys.exit(1)

except Exception as e:
  print(e);
  sys.exit(2)
{'fraction_non_alphanumeric': 0.13181700697854742, 'fraction_numerical': 0.01705867149134143, 'mean_word_length': 2.9773895169578624, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7454978', 'n_tokens_mistral': 1343, 'n_tokens_neox': 1277, 'n_words': 419}