<commit_before>import argparse
import os
import socket
import subprocess as sp

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description = 'LittleJohn Bootstrapper',
        epilog = 'lol moar bitz', add_help = 'How to use',
        prog = 'python littlejohn_setup.py <args>')
    parser.add_argument("-s", "--slaves", required = True,
        help = "Path to a text file containing the list of slaves.")
    parser.add_argument("-m", "--master", required = True,
        help = "Name of the Mesos master node.")
    parser.add_argument("-n", "--namenode", required = True,
        help = "Name of the Hadoop NameNode.")

    args = vars(parser.parse_args())
    if os.path.exists("BUILD"):
        # Remove and start over.
        os.rmdir("BUILD")
    if not os.path.exists("BUILD"):
        sp.call("mkdir -p BUILD/configuration")

    # Get a handle on all the hosts we're dealing with.
    slaves = [line.strip() for line in file(args['slaves'])]
    nodes = slaves
    if args['master'] not in nodes:
        nodes.append(args['master'])
    if args['namenode'] not in nodes:
        nodes.append(args['namenode'])
    fqdn_nodes = [socket.getfqdn(n) for n in nodes]
    ip_nodes = [socket.gethostbyname(n) for n in fqdn_nodes]

    # Step 1: Copy the baremetal script into the BUILD directory.
    sp.call("cp 0-baremetal.sh BUILD/")

    # Step 2: Build the hosts file and copy over the needed bash scripts.
    hosts = """# Sample hosts file, pilfered yet again from Ubuntu 14.04.2.

127.0.0.1\tlocalhost
%s

# The following lines are desirable for IPv6 capable hosts
::1     ip6-localhost ip6-loopback
fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b ip6-localnet
fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b ip6-mcastprefix
fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b ip6-allnodes
fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b ip6-allrouters"""
    lines = ['%s\t%s\t\t%s' % (ip, dn.split(".")[0], dn) for ip, dn in zip(ip_nodes, fqdn_nodes)]
    hosts = hosts % "\n".join(lines)
    os.mkdir("BUILD/configuration/etc")
    f = open("BUILD/configuration/etc/hosts", "w")
    f.write(hosts)
    f.close()
    sp.call("cp templates/bashrc.sh BUILD/configuration/")

    # Step 3: Configure Hadoop.
    hadoop_dir = "BUILD/configuration/hadoop/etc/hadoop"
    sp.call("mkdir -p %s" % hadoop_dir)
    sp.call("cp templates/hadoop/etc/hadoop/hdfs-site.xml %s" % hadoop_dir)
    sp.call("cp templates/hadoop/etc/hadoop/container-executor.cfg %s" % hadoop_dir)
    sp.call("cp templates/hadoop/etc/hadoop/hadoop-env.sh %s" % hadoop_dir)
    f = open("%s/slaves" % hadoop_dir, "w")  # Slaves file.
    f.write("\n".join(slaves))
    f.close()
    f = open("templates/hadoop/etc/hadoop/core-site.xml", "r")
    coresite = f.read()
    f.close()
    f = open("%s/core-site.xml", "w")  # core-site.xml
    f.write(coresite.replace("HADOOP_NAMENODE_HERE", args['namenode']))
    f.close()
    f = open("templates/hadoop/etc/hadoop/mapred-site.xml", "r")
    coresite = f.read()
    f.close()
    f = open("%s/mapred-site.xml", "w")  # mapred-site.xml
    f.write(coresite.replace("HADOOP_NAMENODE_HERE", args['namenode']))
    f.close()

    # Step 4: Configure Spark.
    spark_dir = "BUILD/configuration/spark/conf"
    sp.call("mkdir -p %s" % spark_dir)
    sp.call("cp templates/spark/conf/docker.properties %s" % spark_dir)
    sp.call("cp templates/spark/conf/metrics.properties %s" % spark_dir)
    sp.call("cp templates/spark/conf/spark-env.sh %s" % spark_dir)
    f = open("%s/slaves" % spark_dir, "w")  # Slaves file.
    f.write("\n".join(slaves))
    f.close()
    f = open("templates/spark/conf/spark-defaults.conf", "r")
    defaults = f.read()
    f.close()
    f = open("%s/spark-defaults.conf" % spark_dir, "w")  # spark-defaults.conf
    f.write(defaults.replace("HADOOP_NAMENODE_HERE", args['namenode'])
        .replace("MESOS_MASTER_HERE", args['master']))
    f.close()

    # Step 5: Configure Mesos.
    mesos_dir = "BUILD/configuration/mesos/var/mesos/deploy"
    sp.call("mkdir -p %s" % mesos_dir)
    sp.call("cp templates/mesos/var/mesos/deploy/mesos-deploy-env.sh %s" % mesos_dir)
    sp.call("cp templates/mesos/var/mesos/deploy/mesos-master-env.sh %s" % mesos_dir)
    f = open("%s/slaves" % mesos_dir, "w")  # Slaves file.<commit_msg>Fixed malformed call() subprocess commands (lack of testing will do that).<commit_after>import argparse
import os
import socket
import subprocess as sp

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description = 'LittleJohn Bootstrapper',
        epilog = 'lol moar bitz', add_help = 'How to use',
        prog = 'python littlejohn_setup.py <args>')
    parser.add_argument("-s", "--slaves", required = True,
        help = "Path to a text file containing the list of slaves.")
    parser.add_argument("-m", "--master", required = True,
        help = "Name of the Mesos master node.")
    parser.add_argument("-n", "--namenode", required = True,
        help = "Name of the Hadoop NameNode.")

    args = vars(parser.parse_args())
    if os.path.exists("BUILD"):
        # Remove and start over.
        os.rmdir("BUILD")
    if not os.path.exists("BUILD"):
        sp.call(["mkdir", "-p", "BUILD/configuration"])

    # Get a handle on all the hosts we're dealing with.
    slaves = [line.strip() for line in file(args['slaves'])]
    nodes = slaves
    if args['master'] not in nodes:
        nodes.append(args['master'])
    if args['namenode'] not in nodes:
        nodes.append(args['namenode'])
    fqdn_nodes = [socket.getfqdn(n) for n in nodes]
    ip_nodes = [socket.gethostbyname(n) for n in fqdn_nodes]

    # Step 1: Copy the baremetal script into the BUILD directory.
    sp.call(["cp", "0-baremetal.sh BUILD/"])

    # Step 2: Build the hosts file and copy over the needed bash scripts.
    hosts = """# Sample hosts file, pilfered yet again from Ubuntu 14.04.2.

127.0.0.1\tlocalhost
%s

# The following lines are desirable for IPv6 capable hosts
::1     ip6-localhost ip6-loopback
fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b ip6-localnet
fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b ip6-mcastprefix
fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b ip6-allnodes
fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b ip6-allrouters"""
    lines = ['%s\t%s\t\t%s' % (ip, dn.split(".")[0], dn) for ip, dn in zip(ip_nodes, fqdn_nodes)]
    hosts = hosts % "\n".join(lines)
    os.mkdir("BUILD/configuration/etc")
    f = open("BUILD/configuration/etc/hosts", "w")
    f.write(hosts)
    f.close()
    sp.call(["cp", "templates/bashrc.sh", "BUILD/configuration/"])

    # Step 3: Configure Hadoop.
    hadoop_dir = "BUILD/configuration/hadoop/etc/hadoop"
    sp.call(["mkdir", "-p", hadoop_dir])
    sp.call(["cp", "templates/hadoop/etc/hadoop/hdfs-site.xml", hadoop_dir])
    sp.call(["cp", "templates/hadoop/etc/hadoop/container-executor.cfg", hadoop_dir])
    sp.call(["cp", "templates/hadoop/etc/hadoop/hadoop-env.sh", hadoop_dir])
    f = open("%s/slaves" % hadoop_dir, "w")  # Slaves file.
    f.write("\n".join(slaves))
    f.close()
    f = open("templates/hadoop/etc/hadoop/core-site.xml", "r")
    coresite = f.read()
    f.close()
    f = open("%s/core-site.xml", "w")  # core-site.xml
    f.write(coresite.replace("HADOOP_NAMENODE_HERE", args['namenode']))
    f.close()
    f = open("templates/hadoop/etc/hadoop/mapred-site.xml", "r")
    coresite = f.read()
    f.close()
    f = open("%s/mapred-site.xml", "w")  # mapred-site.xml
    f.write(coresite.replace("HADOOP_NAMENODE_HERE", args['namenode']))
    f.close()

    # Step 4: Configure Spark.
    spark_dir = "BUILD/configuration/spark/conf"
    sp.call(["mkdir", "-p", spark_dir])
    sp.call(["cp", "templates/spark/conf/docker.properties", spark_dir])
    sp.call(["cp", "templates/spark/conf/metrics.properties", spark_dir])
    sp.call(["cp", "templates/spark/conf/spark-env.sh", spark_dir])
    f = open("%s/slaves" % spark_dir, "w")  # Slaves file.
    f.write("\n".join(slaves))
    f.close()
    f = open("templates/spark/conf/spark-defaults.conf", "r")
    defaults = f.read()
    f.close()
    f = open("%s/spark-defaults.conf" % spark_dir, "w")  # spark-defaults.conf
    f.write(defaults.replace("HADOOP_NAMENODE_HERE", args['namenode'])
        .replace("MESOS_MASTER_HERE", args['master']))
    f.close()

    # Step 5: Configure Mesos.
    mesos_dir = "BUILD/configuration/mesos/var/mesos/deploy"
    sp.call(["mkdir", "-p", mesos_dir])
    sp.call(["cp", "templates/mesos/var/mesos/deploy/mesos-deploy-env.sh", mesos_dir])
    sp.call(["cp", "templates/mesos/var/mesos/deploy/mesos-master-env.sh", mesos_dir])
    f = open("%s/slaves" % mesos_dir, "w")  # Slaves file.
{'fraction_non_alphanumeric': 0.1690467897364449, 'fraction_numerical': 0.023685127133402994, 'mean_word_length': 3.4083930399181166, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 14}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2913300', 'n_tokens_mistral': 3350, 'n_tokens_neox': 3192, 'n_words': 656}