<commit_before>import sys
import os
import subprocess

def get_slaves():
    slaves = subprocess.check_output(['cat', '/root/spark-ec2/slaves'])
    return slaves.strip().split('\n')

slaves = get_slaves()
speed = sys.argv[1] + 'Gbit'

commands = ["tc qdisc del dev eth0 root".split(' '),
            "tc qdisc add dev eth0 handle 1: root htb default 11".split(' '),
            "tc class add dev eth0 parent 1: classid 1:1 htb rate 1250Mbps".split(' '),
            ("tc class add dev eth0 parent 1:1 classid 1:11 htb rate " + speed).split(' ')]

for command in commands:
    subprocess.call(command)
<commit_msg>add clear rate limiting to limit_speed<commit_after>import sys
import os
import subprocess

def get_slaves():
    slaves = subprocess.check_output(['cat', '/root/spark-ec2/slaves'])
    return slaves.strip().split('\n')

slaves = get_slaves()
speed = sys.argv[1] + 'Gbit'

if speed == '0Gbit':
    commands = ["tc qdisc del dev eth0 root".split(' ')]
else:
    commands = ["tc qdisc del dev eth0 root".split(' '),
                "tc qdisc add dev eth0 handle 1: root htb default 11".split(' '),
                "tc class add dev eth0 parent 1: classid 1:1 htb rate 1250Mbps".split(' '),
                ("tc class add dev eth0 parent 1:1 classid 1:11 htb rate " + speed).split(' ')]

for command in commands:
    subprocess.call(command)

{'fraction_non_alphanumeric': 0.13895048041389504, 'fraction_numerical': 0.032520325203252036, 'mean_word_length': 3.166153846153846, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2042842', 'n_tokens_mistral': 473, 'n_tokens_neox': 442, 'n_words': 146}