<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import with_statement

import os
import sys
import subprocess

# Deploy the configuration file templates in the spark-ec2/templates directory
# to the root filesystem, substituting variables such as the master hostname,
# ZooKeeper URL, etc as read from the environment.

# Find system memory in KB and compute Spark's default limit from that
mem_command = "cat /proc/meminfo | grep MemTotal | awk '{print $2}'"

master_ram_kb = int(
  os.popen(mem_command).read().strip())
# This is the master's memory. Try to find slave's memory as well
first_slave = os.popen("cat /root/spark-ec2/slaves | head -1").read().strip()

slave_ram_kb = int(subprocess.check_output(
    "ssh -t -o StrictHostKeyChecking=no %s '%s'" %
        (first_slave, mem_command), shell=True).strip())

system_ram_kb = min(slave_ram_kb, master_ram_kb)

system_ram_mb = system_ram_kb / 1024
if system_ram_mb > 20*1024:
  # Leave 3 GB for the OS, HDFS and buffer cache
  spark_mb = system_ram_mb - 3 * 1024
elif system_ram_mb > 10*1024:
  # Leave 2 GB for the OS & co.
  spark_mb = system_ram_mb - 2 * 1024
else:
  # Leave 1.3 GB for the OS & co. Note that this must be more than
  # 1 GB because Mesos leaves 1 GB free and requires 32 MB/task.
  spark_mb = max(512, system_ram_mb - 1300)

template_vars = {
  "master_list": os.getenv("MESOS_MASTERS"),
  "active_master": os.getenv("MESOS_MASTERS").split("\n")[0],
  "slave_list": os.getenv("MESOS_SLAVES"),
  "zoo_list": os.getenv("MESOS_ZOO_LIST"),
  "cluster_url": os.getenv("MESOS_CLUSTER_URL"),
  "hdfs_data_dirs": os.getenv("MESOS_HDFS_DATA_DIRS"),
  "mapred_local_dirs": os.getenv("MESOS_MAPRED_LOCAL_DIRS"),
  "spark_local_dirs": os.getenv("MESOS_SPARK_LOCAL_DIRS"),
  "default_spark_mem": "%dm" % spark_mb
}

template_dir="/root/spark-ec2/templates"

for path, dirs, files in os.walk(template_dir):<commit_msg>Fix slave memory command<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import with_statement

import os
import sys
import subprocess

# Deploy the configuration file templates in the spark-ec2/templates directory
# to the root filesystem, substituting variables such as the master hostname,
# ZooKeeper URL, etc as read from the environment.

# Find system memory in KB and compute Spark's default limit from that
mem_command = "cat /proc/meminfo | grep MemTotal | awk '{print $2}'"

master_ram_kb = int(
  os.popen(mem_command).read().strip())
# This is the master's memory. Try to find slave's memory as well
first_slave = os.popen("cat /root/spark-ec2/slaves | head -1").read().strip()

slave_mem_command = "ssh -t -o StrictHostKeyChecking=no %s %s" %\
        (first_slave, mem_command)
slave_ram_kb = int(os.popen(slave_mem_command).read().strip())

system_ram_kb = min(slave_ram_kb, master_ram_kb)

system_ram_mb = system_ram_kb / 1024
if system_ram_mb > 20*1024:
  # Leave 3 GB for the OS, HDFS and buffer cache
  spark_mb = system_ram_mb - 3 * 1024
elif system_ram_mb > 10*1024:
  # Leave 2 GB for the OS & co.
  spark_mb = system_ram_mb - 2 * 1024
else:
  # Leave 1.3 GB for the OS & co. Note that this must be more than
  # 1 GB because Mesos leaves 1 GB free and requires 32 MB/task.
  spark_mb = max(512, system_ram_mb - 1300)

template_vars = {
  "master_list": os.getenv("MESOS_MASTERS"),
  "active_master": os.getenv("MESOS_MASTERS").split("\n")[0],
  "slave_list": os.getenv("MESOS_SLAVES"),
  "zoo_list": os.getenv("MESOS_ZOO_LIST"),
  "cluster_url": os.getenv("MESOS_CLUSTER_URL"),
  "hdfs_data_dirs": os.getenv("MESOS_HDFS_DATA_DIRS"),
  "mapred_local_dirs": os.getenv("MESOS_MAPRED_LOCAL_DIRS"),
  "spark_local_dirs": os.getenv("MESOS_SPARK_LOCAL_DIRS"),
  "default_spark_mem": "%dm" % spark_mb
}

template_dir="/root/spark-ec2/templates"

for path, dirs, files in os.walk(template_dir):
{'fraction_non_alphanumeric': 0.12334458582186444, 'fraction_numerical': 0.02492858997662945, 'mean_word_length': 4.706666666666667, 'pattern_counts': {'":': 18, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2181388', 'n_tokens_mistral': 1512, 'n_tokens_neox': 1399, 'n_words': 378}