<commit_before>import os
import subprocess
import time

num_nodes = 40
wait_delay = 15 * 60
results_dirname = "results"
partitions = 13
ratios = [(1, 1), (2, 2)] #-p, -q
rates = [125]
backend_mem = "5g"

def run_cmd(cmd):
  subprocess.check_call(cmd, shell=True)

restart_cmd = "./ec2-exp.sh -i ~/.ssh/patkey.pem restart-spark-shark sparrow -m sparrow"
start_cmd = "./ec2-exp.sh -i ~/.ssh/patkey.pem start-shark-tpch sparrow"

for rate in rates:
  for (p, q) in ratios:
    dep_cmd = "./ec2-exp.sh deploy sparrow -g nsdi-patrick -s dev-sparrow-newcode " +\
      "-i ~/.ssh/patkey.pem -p %s -q %s -u %s -v %s --spark-backend-mem=%s" % (
      p, q, partitions, rate, backend_mem)
    run_cmd(dep_cmd)
    run_cmd(restart_cmd)
    run_cmd(start_cmd)
    time.sleep(wait_delay)

    collect_dir = "%s/%s_%s_%s_%s" % (results_dirname, num_nodes, rate, p, q)
    if not os.path.exists(collect_dir):
      os.mkdir(collect_dir)
    collect_cmd = "./ec2-exp.sh -i ~/.ssh/patkey.pem collect-logs sparrow --log-dir=%s" % \
      collect_dir
    run_cmd(collect_cmd)
<commit_msg>Allowing diferent cluster names<commit_after>import os
import subprocess
import time

num_nodes = 20
wait_delay = 20 * 60
results_dirname = "results"
partitions = 6
ratios = [(0, 0), (2, 2)] #-p, -q
rates = [500]
backend_mem = "5g"
cluster_name = "small"

def run_cmd(cmd):
  subprocess.check_call(cmd, shell=True)

restart_cmd = "./ec2-exp.sh -i ~/.ssh/patkey.pem restart-spark-shark %s -m sparrow" % cluster_name
start_cmd = "./ec2-exp.sh -i ~/.ssh/patkey.pem start-shark-tpch %s" % cluster_name

for rate in rates:
  for (p, q) in ratios:
    dep_cmd = "./ec2-exp.sh deploy %s -g patrick-sosp -s dev-sparrow-newcode -i ~/.ssh/patkey.pem -p %s -q %s -u %s -v %s --spark-backend-mem=%s" % (
      cluster_name, p, q, partitions, rate, backend_mem)
    run_cmd(dep_cmd)
    run_cmd(restart_cmd)
    run_cmd(start_cmd)
    time.sleep(wait_delay)

    collect_dir = "%s/%s_%s_%s_%s" % (results_dirname, num_nodes, rate, p, q)
    if not os.path.exists(collect_dir):
      os.mkdir(collect_dir)
    collect_cmd = "./ec2-exp.sh -i ~/.ssh/patkey.pem collect-logs %s --log-dir=%s" % (cluster_name, collect_dir)
    run_cmd(collect_cmd)

    analysis_dir = "%s/analysis" % collect_dir
    if not os.path.exists(analysis_dir):
      os.mkdir(analysis_dir)

    run_cmd("gunzip %s/*.log.gz" % collect_dir)
    run_cmd("python /home/ubuntu/sparrow/src/main/python/parse_tpch_logs.py 600 800 " \
           "/home/ubuntu/sparrow/deploy/ec2/%s/shark_* > /home/ubuntu/sparrow/deploy/ec2/%s/tpch_results" % (collect_dir, analysis_dir))

    run_cmd("/home/ubuntu/sparrow_old/src/main/python/parse_logs.sh start_sec=600 " \
           "end_sec=800 log_dir=/home/ubuntu/sparrow/deploy/ec2/%s/ output_dir=/home/ubuntu/sparrow/deploy/ec2/%s/ > /home/ubuntu/sparrow/deploy/ec2/%s/parse_results" % (
               collect_dir, analysis_dir, analysis_dir))
    run_cmd("gzip %s/*.log" % collect_dir)

{'fraction_non_alphanumeric': 0.16033898305084746, 'fraction_numerical': 0.018983050847457626, 'mean_word_length': 3.2954876273653566, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '642208', 'n_tokens_mistral': 1255, 'n_tokens_neox': 1171, 'n_words': 241}