<commit_before>import subprocess
import getopt
import sys
import time

# Sample rate, 1 minute
SAMPLE=60

def main():
  # Parse the options
  options, args = getopt.getopt(sys.argv[1:], "", ["user=","pass=","host=","dir=","args="])
  
  config = {"user":None,"pass":None,"hosts":[],"dir":None,"args":None}  
  for (flag, val) in options:
    if flag == "--user":
      config["user"] = val
    elif flag == "--pass":
      config["pass"] = val
    elif flag == "--host":
      config["hosts"].append(val)
    elif flag == "--dir":
      config["dir"] = val
    elif flag == "--args":
      config["args"] = val

  # Build the generic command
  cmd = "python remotetest.py --user "+config['user']+" --pass "+config['pass']+" --dir "+config['dir']+" --args '"+config['args']+"' --host "
  
  processes = []
  for host in config["hosts"]:
    # Create an output file for the host
    filename = host+".result.txt"
    print "Storing output for",host,"in",filename
    outfile = open(filename,"w")

    # Launch the process, append the desired host
    print "Staring test on",host
    proc = subprocess.Popen(cmd+host, shell=True, stdout=outfile,stderr=outfile)
    
    # Store this process
    processes.append((host,outfile,proc))
    
    # Wait before starting the next process
    time.sleep(1)

  # Save the time
  start = time.time()

  while True:
    remaining = []
    time.sleep(SAMPLE)
    print "###" # A little space
    print "Checking at:",round(time.time()-start),"seconds..."
    for testrun in processes:
      (host,out,proc) = testrun
      done = not (proc.poll() == None)
      print "Host:",host,"Done:",done
      
      out.flush() # Flush the output
      if not done: remaining.append(testrun) # Store the process
      else: out.close()			     # Close the file handle
    
    # Only keep the non-finished tests
    processes = remaining
    if len(processes) == 0: break	# Stop when all processes finish

  print "Done running all tests!"


if __name__ == '__main__':
  main()

<commit_msg>Added a help flag and a sample rate.<commit_after>import subprocess
import getopt
import sys
import time

# Sample rate, 1 minute
SAMPLE=60

def main():
  global SAMPLE

  # Parse the options
  options, args = getopt.getopt(sys.argv[1:], "", ["help","sample=","user=","pass=","host=","dir=","args="])
  
  config = {"user":None,"pass":None,"hosts":[],"dir":None,"args":None}  
  for (flag, val) in options:
    if flag == "--user":
      config["user"] = val
    elif flag == "--pass":
      config["pass"] = val
    elif flag == "--host":
      config["hosts"].append(val)
    elif flag == "--dir":
      config["dir"] = val
    elif flag == "--args":
      config["args"] = val
    elif flag == "--sample":
      SAMPLE=int(val)
    elif flag == "--help":
      print "Takes the same parameters as remotetest.py with some minor changes."
      print "Does not accept --ip, but allows multiple --host entries."
      print "Takes an optional --sample flag which changes the default sample rate (60 sec)."
      exit()

  # Build the generic command
  cmd = "python remotetest.py --user "+config['user']+" --pass "+config['pass']+" --dir "+config['dir']+" --args '"+config['args']+"' --host "
  
  processes = []
  for host in config["hosts"]:
    # Create an output file for the host
    filename = host+".result.txt"
    print "Storing output for",host,"in",filename
    outfile = open(filename,"w")

    # Launch the process, append the desired host
    print "Staring test on",host
    proc = subprocess.Popen(cmd+host, shell=True, stdout=outfile,stderr=outfile)
    
    # Store this process
    processes.append((host,outfile,proc))
    
    # Wait before starting the next process
    time.sleep(1)

  # Save the time
  start = time.time()

  while True:
    remaining = []
    time.sleep(SAMPLE)
    print "###" # A little space
    print "Checking at:",round(time.time()-start),"seconds..."
    for testrun in processes:
      (host,out,proc) = testrun
      done = not (proc.poll() == None)
      print "Host:",host,"Done:",done
      
      out.flush() # Flush the output
      if not done: remaining.append(testrun) # Store the process
      else: out.close()			     # Close the file handle
    
    # Only keep the non-finished tests
    processes = remaining
    if len(processes) == 0: break	# Stop when all processes finish

  print "Done running all tests!"


if __name__ == '__main__':
  main()


{'fraction_non_alphanumeric': 0.1588447653429603, 'fraction_numerical': 0.00315884476534296, 'mean_word_length': 2.961572832886506, 'pattern_counts': {'":': 22, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4651011', 'n_tokens_mistral': 1390, 'n_tokens_neox': 1361, 'n_words': 422}