<commit_before>#!/usr/bin/env /usr/bin/plcsh
#
# Convert old planetlab3 slice attributes and initscripts to new
# planetlab4 ones.
#
# <NAME> <<EMAIL>>
# Copyright (C) 2006 The Trustees of Princeton University
#
# $Id$
#

import re

# Convert nm_net_{exempt_,}{min,max}_rate (bps) to
# net_{i2_,}{min,max}_rate and net_{i2_,}{min,max}_rate (kbps)
rename = {'nm_net_min_rate': 'net_min_rate',
          'nm_net_max_rate': 'net_max_rate',
          'nm_net_exempt_min_rate': 'net_i2_min_rate',
          'nm_net_exempt_max_rate': 'net_i2_max_rate'}
for slice_attribute in GetSliceAttributes({'name': rename.keys()}):
    id = slice_attribute['slice_attribute_id']
    name = slice_attribute['name']
    slice_id = slice_attribute['slice_id']

    # Convert bps to kbps
    bps = int(slice_attribute['value'])
    kbps = bps / 1000

    # Add the new attribute
    if GetSlices([slice_id]):
        AddSliceAttribute(slice_id, rename[name], str(kbps))

    # Delete the old attribute
    DeleteSliceAttribute(id)

# Convert nm_net_{exempt_,}avg_rate to
# net_{i2_,}max_kbyte and net_{i2_,}thresh_kbyte
rename = {'nm_net_avg_rate': {'max': 'net_max_kbyte',
                              'thresh': 'net_thresh_kbyte'},
          'nm_net_exempt_avg_rate': {'max': 'net_i2_max_kbyte',
                                     'thresh': 'net_i2_thresh_kbyte'}}
for slice_attribute in GetSliceAttributes({'name': rename.keys()}):
    id = slice_attribute['slice_attribute_id']
    name = slice_attribute['name']
    slice_id = slice_attribute['slice_id']

    # Convert bps to 80% and 100% of max bytes per day
    bps = int(slice_attribute['value'])
    max_kbyte = bps * 24 * 60 * 60 / 8 / 1000
    thresh_kbyte = int(0.8 * max_kbyte)

    # Add the new attribute
    if GetSlices([slice_id]):
        AddSliceAttribute(slice_id, rename[name]['max'], str(max_kbyte))
        AddSliceAttribute(slice_id, rename[name]['thresh'], str(thresh_kbyte))

    # Delete the old attribute
    DeleteSliceAttribute(id)

# Convert plc_slice_state
for slice_attribute in GetSliceAttributes({'name': 'plc_slice_state'}):
    id = slice_attribute['slice_attribute_id']
    name = slice_attribute['name']
    slice_id = slice_attribute['slice_id']

    # Add the new attribute
    if GetSlices([slice_id]):
        if slice_attribute['value'] == "suspended":
            AddSliceAttribute(slice_id, 'enabled', "0")
        else:
            AddSliceAttribute(slice_id, 'enabled', "1")

    # Delete the old attribute
    DeleteSliceAttribute(id)
    
# Straight renames
rename = {'nm_cpu_share': 'cpu_share',
          'nm_disk_quota': 'disk_max',
          'nm_net_share': 'net_share',
          'nm_net_exempt_share': 'net_i2_share',
          'nm_net_max_byte': 'net_max_kbyte',
          'nm_net_max_thresh_byte': 'net_thresh_kbyte',
          'nm_net_max_exempt_byte': 'net_i2_max_kbyte',
          'nm_net_max_thresh_exempt_byte': 'net_i2_thresh_kbyte'}
for slice_attribute in GetSliceAttributes({'name': rename.keys()}):
    id = slice_attribute['slice_attribute_id']
    name = slice_attribute['name']
    slice_id = slice_attribute['slice_id']

    # Pass straight through
    value = slice_attribute['value']

    # Add the new attribute
    if GetSlices([slice_id]):
        AddSliceAttribute(slice_id, rename[name], value)

    # Delete the old attribute
    DeleteSliceAttribute(id)

# Delete _deleted and deprecated slice attributes and types
for attribute_type in GetSliceAttributeTypes():
    id = attribute_type['attribute_type_id']
    name = attribute_type['name']

    if name == 'general_prop_share' or \
       re.match('nm_', name) or \
       re.search('_deleted$', name):
        DeleteSliceAttributeType(id)
        # N.B. Automatically deletes all slice attributes of this type

initscripts = dict([(initscript['initscript_id'], initscript) for initscript in [{'initscript_id': 8, 'script': '<KEY> 'name': 'arizona_stork_2', 'encoding': 'base64'}, {'initscript_id': 9, 'script': 'IyEvYmluL2Jhc2gNCmNkIC8NCnJtIC1mIHN0YXJ0X3B1cnBsZQ0Kd2dldCBodHRwOi8vd3d3LmNzLnByaW5jZXRvbi5lZHUvfmRlaXNlbnN0L3B1cnBsZS9zdGFydF9wdXJwbGUNCmNobW9kIDc1NSBzdGFydF9wdXJwbGUNCnN1IHByaW5jZXRvbl9wdXJwbGUgLWMgJy4vc3RhcnRfcHVycGxlJw0K', 'name': 'princeton_purple', 'encoding': 'base64'}, {'initscript_id': 6, 'script': '<KEY> 'name': 'princeton_hello_stork', 'encoding': 'base64'}, {'initscript_id': 10, 'script': '<KEY> 'name': 'princeton_hello', 'encoding': 'base64'}]])

# Convert plc_initscript.initscript_id to raw initscript attribute
for slice_attribute in GetSliceAttributes({'name': 'plc_initscript'}):
    id = slice_attribute['slice_attribute_id']
    slice_id = slice_attribute['slice_id']
    initscript_id = int(slice_attribute['value'])

    # Delete old attribute
    DeleteSliceAttribute(id)

    if initscript_id not in initscripts:
        print "Warning: Missing initscript %d" % initscript_id
        continue<commit_msg>- add Proper ops<commit_after>#!/usr/bin/env /usr/bin/plcsh
#
# Convert old planetlab3 slice attributes and initscripts to new
# planetlab4 ones.
#
# <NAME> <<EMAIL>>
# Copyright (C) 2006 The Trustees of Princeton University
#
# $Id: slice_attributes.py,v 1.1 2007/02/02 23:55:31 mlhuang Exp $
#

import re

# Convert nm_net_{exempt_,}{min,max}_rate (bps) to
# net_{i2_,}{min,max}_rate and net_{i2_,}{min,max}_rate (kbps)
rename = {'nm_net_min_rate': 'net_min_rate',
          'nm_net_max_rate': 'net_max_rate',
          'nm_net_exempt_min_rate': 'net_i2_min_rate',
          'nm_net_exempt_max_rate': 'net_i2_max_rate'}
for slice_attribute in GetSliceAttributes({'name': rename.keys()}):
    id = slice_attribute['slice_attribute_id']
    name = slice_attribute['name']
    slice_id = slice_attribute['slice_id']

    # Convert bps to kbps
    bps = int(slice_attribute['value'])
    kbps = bps / 1000

    # Add the new attribute
    if GetSlices([slice_id]):
        AddSliceAttribute(slice_id, rename[name], str(kbps))

    # Delete the old attribute
    DeleteSliceAttribute(id)

# Convert nm_net_{exempt_,}avg_rate to
# net_{i2_,}max_kbyte and net_{i2_,}thresh_kbyte
rename = {'nm_net_avg_rate': {'max': 'net_max_kbyte',
                              'thresh': 'net_thresh_kbyte'},
          'nm_net_exempt_avg_rate': {'max': 'net_i2_max_kbyte',
                                     'thresh': 'net_i2_thresh_kbyte'}}
for slice_attribute in GetSliceAttributes({'name': rename.keys()}):
    id = slice_attribute['slice_attribute_id']
    name = slice_attribute['name']
    slice_id = slice_attribute['slice_id']

    # Convert bps to 80% and 100% of max bytes per day
    bps = int(slice_attribute['value'])
    max_kbyte = bps * 24 * 60 * 60 / 8 / 1000
    thresh_kbyte = int(0.8 * max_kbyte)

    # Add the new attribute
    if GetSlices([slice_id]):
        AddSliceAttribute(slice_id, rename[name]['max'], str(max_kbyte))
        AddSliceAttribute(slice_id, rename[name]['thresh'], str(thresh_kbyte))

    # Delete the old attribute
    DeleteSliceAttribute(id)

# Convert plc_slice_state
for slice_attribute in GetSliceAttributes({'name': 'plc_slice_state'}):
    id = slice_attribute['slice_attribute_id']
    name = slice_attribute['name']
    slice_id = slice_attribute['slice_id']

    # Add the new attribute
    if GetSlices([slice_id]):
        if slice_attribute['value'] == "suspended":
            AddSliceAttribute(slice_id, 'enabled', "0")
        else:
            AddSliceAttribute(slice_id, 'enabled', "1")

    # Delete the old attribute
    DeleteSliceAttribute(id)
    
# Straight renames
rename = {'nm_cpu_share': 'cpu_share',
          'nm_disk_quota': 'disk_max',
          'nm_net_share': 'net_share',
          'nm_net_exempt_share': 'net_i2_share',
          'nm_net_max_byte': 'net_max_kbyte',
          'nm_net_max_thresh_byte': 'net_thresh_kbyte',
          'nm_net_max_exempt_byte': 'net_i2_max_kbyte',
          'nm_net_max_thresh_exempt_byte': 'net_i2_thresh_kbyte'}
for slice_attribute in GetSliceAttributes({'name': rename.keys()}):
    id = slice_attribute['slice_attribute_id']
    name = slice_attribute['name']
    slice_id = slice_attribute['slice_id']

    # Pass straight through
    value = slice_attribute['value']

    # Add the new attribute
    if GetSlices([slice_id]):
        AddSliceAttribute(slice_id, rename[name], value)

    # Delete the old attribute
    DeleteSliceAttribute(id)

# Delete _deleted and deprecated slice attributes and types
for attribute_type in GetSliceAttributeTypes():
    id = attribute_type['attribute_type_id']
    name = attribute_type['name']

    if name == 'general_prop_share' or \
       re.match('nm_', name) or \
       re.search('_deleted$', name):
        DeleteSliceAttributeType(id)
        # N.B. Automatically deletes all slice attributes of this type

# Add Proper ops
proper_ops = [
    # give Stork permission to mount and unmount client dirs
    ('arizona_stork', 'mount_dir'),
    ('arizona_stork', 'set_file_flags pass, "1"'),
    ('arizona_stork', 'set_file_flags_list "1"'),
    ('arizona_stork', 'bind_socket sockname=64?:*'),
    ('arizona_stork2', 'mount_dir'),
    ('arizona_stork2', 'set_file_flags pass, "1"'),
    ('arizona_stork2', 'set_file_flags_list "1"'),
    ('arizona_stork2', 'bind_socket sockname=64?:*'),

    # give CoMon the necessary permissions to run slicestat
    ('princeton_slicestat', 'exec "root", pass, "/usr/local/planetlab/bin/pl-ps", none'),
    ('princeton_slicestat', 'exec "root", pass, "/usr/sbin/vtop", "bn1", none'),
    ('princeton_slicestat', 'open_file file=/proc/virtual/*/cacct'),
    ('princeton_slicestat', 'open_file file=/proc/virtual/*/limit'),
    ('princeton_comon', 'open_file file=/var/log/secure'),
    ('princeton_comon', 'exec "root", pass, "/bin/df", "/vservers", none'),

    # give pl_slicedir access to /etc/passwd
    ('pl_slicedir', 'open_file pass, "/etc/passwd"'),

    # nyu_d are building a DNS demux so give them access to port 53
    ('nyu_d', 'bind_socket'),
    ('nyu_oasis', 'bind_socket'),

    # QA slices need to be able to create and delete bind-mounts
    ('pl_qa_0', 'mount_dir'),
    ('pl_qa_1', 'mount_dir'),

    # irb_snort needs packet sockets for tcpdump
    ('irb_snort', 'create_socket'),

    # uw_ankur is using netlink sockets to do the same thing as netflow
    ('uw_ankur', 'create_socket'),

    # cornell_codons gets access to port 53 for now
    ('cornell_codons', 'create_socket'),

    # give <NAME>'s conf-monitor service read-only access to root fs
    # and the ability to run df
    ('idsl_monitor', 'mount_dir "root:/", pass, "ro"'),
    ('idsl_monitor', 'unmount'),
    ('idsl_monitor', 'exec "root", pass, "/bin/df", "-P", "/", "/vservers", none'),

    # give Shark access to port 111 to run portmap
    # and port 955 to run mount
    ('nyu_shkr', 'bind_socket'),
    ('nyu_shkr', 'mount_dir "nfs:**:**"'),
    ('nyu_shkr', 'exec "root", pass, "/bin/umount", "-l", "/vservers/nyu_shkr/**", none'),

    # give tsinghua_lgh access to restricted ports
    ('tsinghua_lgh', 'bind_socket'),

    # CoDeeN needs port 53 too
    ('princeton_codeen', 'bind_socket sockname=53:*'),

    # give ucin_load access to /var/log/wtmp
    ('ucin_load', 'open_file file=/var/log/wtmp*'),

    # give google_highground permission to bind port 81 (and raw sockets)
    ('google_highground', 'bind_socket'),

    # pl_conf needs access to port 814
    ('pl_conf', 'bind_socket sockname=814:*'),
    ('pl_conf', 'open file=/home/*/.ssh/authorized_keys'),

    # give princeton_visp permission to read all packets sent through the
    # tap0 device
    ('princeton_visp', 'open file=/dev/net/tun, flags=rw'),

    # The PLB group needs the BGP port
    ('princeton_iias', 'bind_socket sockname=179:*'),
    ('princeton_visp', 'bind_socket sockname=179:*'),
    ('mit_rcp', 'bind_socket sockname=179:*'),
    ('princeton_bgpmux', 'bind_socket sockname=179:*'),
    ('princeton_bgpmux2', 'bind_socket sockname=179:*'),

    # PL-VINI group
    ('mit_rcp', 'exec "root", pass, "/usr/bin/chrt"'),
    ('princeton_iias', 'exec "root", pass, "/usr/bin/chrt"'),

    # Tycoon needs access to /etc/passwd to determine Slicename->XID mappings
    ('hplabs_tycoon_aucd', 'open_file file=/etc/passwd'),
]

for slice, op in proper_ops:
    try:
        AddSliceAttribute(slice, 'proper_op', op)
    except Exception, err:
        print "Warning: %s:" % slice, err

initscripts = dict([(initscript['initscript_id'], initscript) for initscript in [{'initscript_id': 8, 'script': '<KEY> 'name': 'arizona_stork_2', 'encoding': 'base64'}, {'initscript_id': 9, 'script': '<KEY>', 'name': 'princeton_purple', 'encoding': 'base64'}, {'initscript_id': 6, 'script': '<KEY> 'name': 'princeton_hello_stork', 'encoding': 'base64'}, {'initscript_id': 10, 'script': '<KEY> 'name': 'princeton_hello', 'encoding': 'base64'}]])

# Convert plc_initscript.initscript_id to raw initscript attribute
for slice_attribute in GetSliceAttributes({'name': 'plc_initscript'}):
    id = slice_attribute['slice_attribute_id']
    slice_id = slice_attribute['slice_id']
    initscript_id = int(slice_attribute['value'])

    # Delete old attribute
    DeleteSliceAttribute(id)

    if initscript_id not in initscripts:
        print "Warning: Missing initscript %d" % initscript_id
        continue
{'fraction_non_alphanumeric': 0.13768389287061486, 'fraction_numerical': 0.015767634854771784, 'mean_word_length': 4.0906298003072195, 'pattern_counts': {'":': 2, '<': 17, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1430299', 'n_tokens_mistral': 4758, 'n_tokens_neox': 4588, 'n_words': 1008}