<commit_before>from __future__ import print_function
from fabric.api import run, env, execute, task
from fabric.context_managers import cd

import argparse
import os.path
import sys

import yaml

parser = argparse.ArgumentParser(
  description="Remotely run the various Docker images in storm-docker"
)
parser.add_argument("--zk", "--zookeeper", action="store_true",
  dest="zookeeper", help="Run Zookeeper Docker images"
)
parser.add_argument("--supervisor", action="store_true",
  dest="supervisor", help="Run Storm supervisor Docker images"
)

# Use settings in SSH config file
env.use_ssh_config = True

def _main():
  yaml_file_path = os.path.join("config", "storm-setup.yaml")
  if not os.path.exists(yaml_file_path):
    print("{} does not exist. Exiting.".format(yaml_file_path), file=sys.stderr)
    sys.exit(1)
  with open(yaml_file_path, "r") as f:
    d = yaml.load(f.read())
  args = parser.parse_args()
  if args.zookeeper:
    zk_server_list = d["storm.yaml"]["storm.zookeeper.servers"]
    execute(_run_docker_component, "zookeeper", hosts=zk_server_list)
  if args.supervisor:
    execute(
      _run_docker_component, "supervisor",
      hosts=d["storm.supervisor.hosts"]
    )

@task
def _run_docker_component(component):
  with cd("$HOME"):
    with cd("storm-docker"):
      run("./destroy-storm.sh {}".format(component))
      run("./start-storm.sh {}".format(component))

if __name__ == "__main__":
  _main()<commit_msg>remote.py - set warn_only=True when running destroy-storm.sh script<commit_after>from __future__ import print_function
from fabric.api import run, env, execute, task
from fabric.context_managers import cd

import argparse
import os.path
import sys

import yaml

parser = argparse.ArgumentParser(
  description="Remotely run the various Docker images in storm-docker"
)
parser.add_argument("--zk", "--zookeeper", action="store_true",
  dest="zookeeper", help="Run Zookeeper Docker images"
)
parser.add_argument("--supervisor", action="store_true",
  dest="supervisor", help="Run Storm supervisor Docker images"
)

# Use settings in SSH config file
env.use_ssh_config = True

def _main():
  yaml_file_path = os.path.join("config", "storm-setup.yaml")
  if not os.path.exists(yaml_file_path):
    print("{} does not exist. Exiting.".format(yaml_file_path), file=sys.stderr)
    sys.exit(1)
  with open(yaml_file_path, "r") as f:
    d = yaml.load(f.read())
  args = parser.parse_args()
  if args.zookeeper:
    zk_server_list = d["storm.yaml"]["storm.zookeeper.servers"]
    execute(_run_docker_component, "zookeeper", hosts=zk_server_list)
  if args.supervisor:
    execute(
      _run_docker_component, "supervisor",
      hosts=d["storm.supervisor.hosts"]
    )

@task
def _run_docker_component(component):
  with cd("$HOME"):
    with cd("storm-docker"):
      run("./destroy-storm.sh {}".format(component), warn_only=True)
      run("./start-storm.sh {}".format(component))

if __name__ == "__main__":
  _main()
{'fraction_non_alphanumeric': 0.13903924221921515, 'fraction_numerical': 0.0006765899864682003, 'mean_word_length': 4.741747572815534, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5193100', 'n_tokens_mistral': 1002, 'n_tokens_neox': 979, 'n_words': 224}