<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-


import sys
import math

def statistics ( f ):
  sopt = vopt = salg = valg = scen = vcen = 0
  num = 0
  for l in open( f ):
    num += 1
    if num == 1:
      continue
    opt, alg, cen = l.split()[2:] #[:-1]
    sopt += int( opt )
    vopt += int( opt ) * int( opt )
    salg += int( alg )
    valg += int( alg ) * int( alg )
    scen += int( cen )
    vcen += int( cen ) * int( cen )
  
  print( "algo\taverage\t\tvariance" )
  print( "opt\t{0}\t{1}".format( sopt / float( num ), math.sqrt( vopt / float( num ) - ( sopt / float( num ) ) ** 2 ) ) )
  print( "new\t{0}\t{1}".format( salg / float( num ), math.sqrt( valg / float( num ) - ( salg / float( num ) ) ** 2 ) ) )
  print( "cen\t{0}\t{1}".format( scen / float( num ), math.sqrt( vcen / float( num ) - ( scen / float( num ) ) ** 2 ) ) )
    

if __name__ == '__main__':
	f = sys.argv[1]
	statistics( f )
<commit_msg>modified script<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-


import sys
import math

def statistics ( f ):
  sopt = vopt = salg = valg = scen = vcen = 0
  num = 0
  for l in open( f ):
    num += 1
    if num == 1:
      continue
    (opt, alg, cen), rest = l.split()[2:5], l.split()[5:]
    sopt += int( opt )
    vopt += int( opt ) * int( opt )
    salg += int( alg )
    valg += int( alg ) * int( alg )
    scen += int( cen )
    vcen += int( cen ) * int( cen )

  print( f )  
  print( "algo\taverage\t\tvariance" )
  print( "opt\t{0}\t{1}".format( sopt / float( num ), math.sqrt( vopt / float( num ) - ( sopt / float( num ) ) ** 2 ) ) )
  print( "new\t{0}\t{1}".format( salg / float( num ), math.sqrt( valg / float( num ) - ( salg / float( num ) ) ** 2 ) ) )
  print( "cen\t{0}\t{1}".format( scen / float( num ), math.sqrt( vcen / float( num ) - ( scen / float( num ) ) ** 2 ) ) )
    

if __name__ == '__main__':
	f = sys.argv[1]
	statistics( f )

{'fraction_non_alphanumeric': 0.2061909758656873, 'fraction_numerical': 0.018363064008394544, 'mean_word_length': 2.2104377104377106, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2161107', 'n_tokens_mistral': 798, 'n_tokens_neox': 776, 'n_words': 220}