<commit_before>import os
import argparse

def main():
  parser = argparse.ArgumentParser(description='TODO')
  parser.add_argument('-ho', '--host_dir', required=True)
  parser.add_argument('-s', '--submit_script', required=True)
  parser.add_argument('-ex', '--executable', required=True)
  args = parser.parse_args()

  ndata = [10^4, 10^5, 10^6, 10^7, 10^8]
  nrepeat = [10^2, 10^3, 10^4, 10^5]

  machines = [2,4,8,16,31]

  for data in ndata:
    for repeat in nrepeat:
      for machine in machines:
        host_file = os.path.join(args.host_dir, 'host%d' % machine)
        cmd = 'python %s %d %s %s %d %d' % (args.submit_script, machine, host_file, args.executable, data, repeat)
        print 'data=%d, repeat=%d, machine=%d' % (data, repeat, machine)
        os.system(cmd)

if __name__ == "__main__":
  main()<commit_msg>speed runner<commit_after>import os
import argparse

def main():
  parser = argparse.ArgumentParser(description='TODO')
  parser.add_argument('-ho', '--host_dir', required=True)
  parser.add_argument('-s', '--submit_script', required=True)
  parser.add_argument('-ex', '--executable', required=True)
  args = parser.parse_args()

  ndata = [10^4, 10^5, 10^6, 10^7, 10^8]
  nrepeat = [10^2, 10^3, 10^4, 10^5]

  machines = [2,4,8,16,31]

  for data in ndata:
    for repeat in nrepeat:
      for machine in machines:
        host_file = os.path.join(args.host_dir, 'hosts%d' % machine)
        cmd = 'python %s %d %s %s %d %d' % (args.submit_script, machine, host_file, args.executable, data, repeat)
        print 'data=%d, repeat=%d, machine=%d' % (data, repeat, machine)
        os.system(cmd)

if __name__ == "__main__":
  main()
{'fraction_non_alphanumeric': 0.171875, 'fraction_numerical': 0.040865384615384616, 'mean_word_length': 3.8401162790697674, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1162995', 'n_tokens_mistral': 659, 'n_tokens_neox': 619, 'n_words': 148}