<commit_before>"""
StarCluster plugin to perform per-node configuration
for grappa
See: http://sampa.cs.washington.edu/grappa

"""
from starcluster import clustersetup
from starcluster.logger import log
import sys
import subprocess

class SetShmmax(clustersetup.DefaultClusterSetup):
    """
    This plugin sets shmmax to the maximum memory available on
    each node in the cluster unless otherwise specified by shmmaxkB

    [plugin set_shmmax]
    setup_class = softxmt.starcluster.plugins.config_grappa.SetShmmax
optionally:
    shmmaxkB = 123456789

    """
    def __init__(self, shmmaxkB=0):
        super(SetShmmax, self).__init__()
        self.shmmaxkB = shmmaxkB

    def run(self, nodes, master, user, user_shell, volumes):
        shmmax = 1024*self.shmmaxkB
        for node in nodes:
            if shmmax > 0:
                self.pool.simple_job(node.ssh.execute, ('sysctl -w kernel.shmmax=%d' %shmmax), jobid=node.alias)           
            else:
                    self.pool.simple_job(node.ssh.execute, ("sysctl -w kernel.shmmax=`grep MemTotal /proc/meminfo | awk '{print $2*1024}'`"), jobid=node.alias)

        self.pool.wait(len(nodes))

    def on_add_node(self, new_node, nodes, master, user, user_shell, volumes):
        shmmax = 1024*self.shmmaxkB
        if shmmax > 0:
            self.pool.simple_job(new_node.ssh.execute, ('sysctl -w kernel.shmmax=%d' %shmmax), jobid=node.alias)           
        else:
            self.pool.simple_job(new_node.ssh.execute, ("sysctl -w kernel.shmmax=`grep MemTotal /proc/meminfo | awk '{print 1024*$2}'`"), jobid=node.alias)

    def on_remove_node(self, node, nodes, master, user, user_shell, volumes):
        pass<commit_msg>add kernel.shmall setting to shmmax Starcluster plugin<commit_after>"""
StarCluster plugin to perform per-node configuration
for grappa
See: http://sampa.cs.washington.edu/grappa

"""
from starcluster import clustersetup
from starcluster.logger import log
import sys
import subprocess

class SetShmmax(clustersetup.DefaultClusterSetup):
    """
    This plugin sets shmmax to the maximum memory available on
    each node in the cluster unless otherwise specified by shmmaxkB

    [plugin set_shmmax]
    setup_class = softxmt.starcluster.plugins.config_grappa.SetShmmax
optionally:
    shmmaxkB = 123456789

    """
    def __init__(self, shmmaxkB=0):
        super(SetShmmax, self).__init__()
        self.shmmaxkB = shmmaxkB

    def run(self, nodes, master, user, user_shell, volumes):
        shmmax = 1024*self.shmmaxkB
        for node in nodes:
            if shmmax > 0:
                self.pool.simple_job(node.ssh.execute, ('sysctl -w kernel.shmmax=%d' %shmmax), jobid=node.alias)
                self.pool.simple_job(node.ssh.execute, ('sysctl -w kernel.shmall=%d' %shmmax/4096), jobid=node.alias)
            else:
                self.pool.simple_job(node.ssh.execute, ("sysctl -w kernel.shmmax=`grep MemTotal /proc/meminfo | awk '{print rshift($2*1024,1)}'`"), jobid=node.alias)
                self.pool.simple_job(node.ssh.execute, ("sysctl -w kernel.shmall=`grep MemTotal /proc/meminfo | awk '{print rshift($2*1024,13)}'`"), jobid=node.alias)

        self.pool.wait(len(nodes))

    def on_add_node(self, new_node, nodes, master, user, user_shell, volumes):
        shmmax = 1024*self.shmmaxkB
        if shmmax > 0:
            self.pool.simple_job(new_node.ssh.execute, ('sysctl -w kernel.shmmax=%d' %shmmax), jobid=node.alias)           
            self.pool.simple_job(new_node.ssh.execute, ('sysctl -w kernel.shmall=%d' %shmmax/4096), jobid=node.alias)
        else:
            self.pool.simple_job(new_node.ssh.execute, ("sysctl -w kernel.shmmax=`grep MemTotal /proc/meminfo | awk '{print rshift($2*1024,1)}'`"), jobid=node.alias)
            self.pool.simple_job(new_node.ssh.execute, ("sysctl -w kernel.shmall=`grep MemTotal /proc/meminfo | awk '{print rshift($2*1024,13)}'`"), jobid=node.alias)

    def on_remove_node(self, node, nodes, master, user, user_shell, volumes):
        pass
{'fraction_non_alphanumeric': 0.12170233947237431, 'fraction_numerical': 0.020905923344947737, 'mean_word_length': 3.164766839378238, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3814908', 'n_tokens_mistral': 1446, 'n_tokens_neox': 1336, 'n_words': 301}