<commit_before>
def _sanitize_arg(arg):
    if " " in arg:
        arg = '"%s"' % arg

    return arg

def lnet_load():
    return "modprobe lnet"

def lnet_unload():
    return "lustre_rmmod || lctl net down && lustre_rmmod"

def lnet_start():
    return "lctl net up"

def lnet_stop():
    return "lctl net down || (lustre_rmmod; lctl net down)"

def mount(device="", dir=""):
    if len(device) > 0 and len(dir) > 0:
        return "mount -t lustre %s %s" % (device, dir)
    else:
        raise ValueError, "mount() needs both device and dir"

def umount(device="", dir=""):
    if len(device) > 0 and len(dir) > 0:
        return "umount %s || umount %s" % (device, dir)
    elif len(device) > 0:
        return "umount %s" % device
    elif len(dir) > 0:
        return "umount %s" % dir
    else:
        return "umount -a -tlustre"

def tunefs(device="", target_types=(), mgsnode=(), fsname="", failnode=(),
           servicenode=(), param={}, index="", comment="", mountfsoptions="",
           network=(), erase_params=False, nomgs=False, writeconf=False,
           dryrun=False, verbose=False, quiet=False):

    # freeze a view of the namespace before we start messing with it
    args = locals()
    types = ""
    options = ""

    tuple_options = "target_types mgsnode failnode servicenode network".split()
    for name in tuple_options:
        arg = args[name]
        # ensure that our tuple arguments are always tuples, and not strings
        if not hasattr(arg, "__iter__"):
            arg = (arg,)

        if name == "target_types":
            for type in arg:
                types += "--%s " % type
        else:
            if len(arg) > 0:
                options += "--%s=%s " % (name, ",".join(arg))

    flag_options = {
        'erase_params': '--erase-params',
        'nomgs': '--nomgs',
        'writeconf': '--writeconf',
        'dryrun': '--dryrun',
        'verbose': '--verbose',
        'quiet': '--quiet',
    }
    for arg in flag_options:
        if args[arg]:
            options += "%s " % flag_options[arg]

    dict_options = "param".split()
    for name in dict_options:
        arg = args[name]
        for key in arg:
            if arg[key] is not None:
                options += "--%s %s=%s " % (name, key, _sanitize_arg(arg[key]))

    # everything else
    handled = set(flag_options.keys() + tuple_options + dict_options)
    for name in set(args.keys()) - handled:
        if name == "device":
            continue
        value = args[name]
        if value != '':
            options += "--%s=%s " % (name, _sanitize_arg(value))

    # NB: Use $PATH instead of relying on hard-coded paths
    cmd = "tunefs.lustre %s %s %s" % (types, options, device)

    return ' '.join(cmd.split())

def mkfs(device="", target_types=(), mgsnode=(), fsname="", failnode=(),
         servicenode=(), param={}, index="", comment="", mountfsoptions="",
         network=(), backfstype="", device_size="", mkfsoptions="",
         reformat=False, stripe_count_hint="", iam_dir=False,
         dryrun=False, verbose=False, quiet=False):

    # freeze a view of the namespace before we start messing with it
    args = locals()
    types = ""
    options = ""

    tuple_options = "target_types mgsnode failnode servicenode network".split()
    for name in tuple_options:
        arg = args[name]
        # ensure that our tuple arguments are always tuples, and not strings
        if not hasattr(arg, "__iter__"):
            arg = (arg,)

        if name == "target_types":
            for type in arg:
                types += "--%s " % type
        else:
            if len(arg) > 0:
                options += "--%s=%s " % (name, ",".join(arg))
                
    flag_options = {
        'dryrun': '--dryrun',
        'reformat': '--reformat',
        'iam_dir': '--iam-dir',
        'verbose': '--verbose',
        'quiet': '--quiet',
    }
    for arg in flag_options:
        if args[arg]:
            options += "%s " % flag_options[arg]

    dict_options = "param".split()
    for name in dict_options:
        arg = args[name]
        for key in arg:
            if arg[key] is not None:
                options += "--%s %s=%s " % (name, key, _sanitize_arg(arg[key]))

    # everything else
    handled = set(flag_options.keys() + tuple_options + dict_options)
    for name in set(args.keys()) - handled:
        if name == "device":
            continue
        value = args[name]
        if value != '':
            options += "--%s=%s " % (name, _sanitize_arg(value))

    # NB: Use $PATH instead of relying on hard-coded paths
    cmd = "mkfs.lustre %s %s %s" % (types, options, device)

    return ' '.join(cmd.split())<commit_msg>Slightly improve __sanitize() to quote any whitespace<commit_after>import re

def __sanitize_arg(arg):
    if re.search(r'\s', arg):
        arg = '"%s"' % arg

    return arg

def lnet_load():
    return "modprobe lnet"

def lnet_unload():
    return "lustre_rmmod || lctl net down && lustre_rmmod"

def lnet_start():
    return "lctl net up"

def lnet_stop():
    return "lctl net down || (lustre_rmmod; lctl net down)"

def mount(device="", dir=""):
    if len(device) > 0 and len(dir) > 0:
        return "mount -t lustre %s %s" % (device, dir)
    else:
        raise ValueError, "mount() needs both device and dir"

def umount(device="", dir=""):
    if len(device) > 0 and len(dir) > 0:
        return "umount %s || umount %s" % (device, dir)
    elif len(device) > 0:
        return "umount %s" % device
    elif len(dir) > 0:
        return "umount %s" % dir
    else:
        return "umount -a -tlustre"

def tunefs(device="", target_types=(), mgsnode=(), fsname="", failnode=(),
           servicenode=(), param={}, index="", comment="", mountfsoptions="",
           network=(), erase_params=False, nomgs=False, writeconf=False,
           dryrun=False, verbose=False, quiet=False):

    # freeze a view of the namespace before we start messing with it
    args = locals()
    types = ""
    options = ""

    tuple_options = "target_types mgsnode failnode servicenode network".split()
    for name in tuple_options:
        arg = args[name]
        # ensure that our tuple arguments are always tuples, and not strings
        if not hasattr(arg, "__iter__"):
            arg = (arg,)

        if name == "target_types":
            for type in arg:
                types += "--%s " % type
        else:
            if len(arg) > 0:
                options += "--%s=%s " % (name, ",".join(arg))

    flag_options = {
        'erase_params': '--erase-params',
        'nomgs': '--nomgs',
        'writeconf': '--writeconf',
        'dryrun': '--dryrun',
        'verbose': '--verbose',
        'quiet': '--quiet',
    }
    for arg in flag_options:
        if args[arg]:
            options += "%s " % flag_options[arg]

    dict_options = "param".split()
    for name in dict_options:
        arg = args[name]
        for key in arg:
            if arg[key] is not None:
                options += "--%s %s=%s " % (name, key, __sanitize_arg(arg[key]))

    # everything else
    handled = set(flag_options.keys() + tuple_options + dict_options)
    for name in set(args.keys()) - handled:
        if name == "device":
            continue
        value = args[name]
        if value != '':
            options += "--%s=%s " % (name, __sanitize_arg(value))

    # NB: Use $PATH instead of relying on hard-coded paths
    cmd = "tunefs.lustre %s %s %s" % (types, options, device)

    return ' '.join(cmd.split())

def mkfs(device="", target_types=(), mgsnode=(), fsname="", failnode=(),
         servicenode=(), param={}, index="", comment="", mountfsoptions="",
         network=(), backfstype="", device_size="", mkfsoptions="",
         reformat=False, stripe_count_hint="", iam_dir=False,
         dryrun=False, verbose=False, quiet=False):

    # freeze a view of the namespace before we start messing with it
    args = locals()
    types = ""
    options = ""

    tuple_options = "target_types mgsnode failnode servicenode network".split()
    for name in tuple_options:
        arg = args[name]
        # ensure that our tuple arguments are always tuples, and not strings
        if not hasattr(arg, "__iter__"):
            arg = (arg,)

        if name == "target_types":
            for type in arg:
                types += "--%s " % type
        else:
            if len(arg) > 0:
                options += "--%s=%s " % (name, ",".join(arg))
                
    flag_options = {
        'dryrun': '--dryrun',
        'reformat': '--reformat',
        'iam_dir': '--iam-dir',
        'verbose': '--verbose',
        'quiet': '--quiet',
    }
    for arg in flag_options:
        if args[arg]:
            options += "%s " % flag_options[arg]

    dict_options = "param".split()
    for name in dict_options:
        arg = args[name]
        for key in arg:
            if arg[key] is not None:
                options += "--%s %s=%s " % (name, key, __sanitize_arg(arg[key]))

    # everything else
    handled = set(flag_options.keys() + tuple_options + dict_options)
    for name in set(args.keys()) - handled:
        if name == "device":
            continue
        value = args[name]
        if value != '':
            options += "--%s=%s " % (name, __sanitize_arg(value))

    # NB: Use $PATH instead of relying on hard-coded paths
    cmd = "mkfs.lustre %s %s %s" % (types, options, device)

    return ' '.join(cmd.split())
{'fraction_non_alphanumeric': 0.15248677248677248, 'fraction_numerical': 0.0016931216931216932, 'mean_word_length': 2.1704126132170414, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3595865', 'n_tokens_mistral': 3063, 'n_tokens_neox': 2982, 'n_words': 882}