<commit_before># Zfs helper functions

from subprocess import check_output, CalledProcessError

ZFS_COMMAND="zfs"
REMOTE_COMMAND="ssh"
REMOTE_COMMAND_PARAMS="-c"
REMOTE_COMMAND_PARAMS2="arcfour,blowfish-cbc"
GLOBAL_PARAMETERS="-H"
DRY="echo"

# this function aims to get a list of a remote dataset
def list(host, dataset, type='dataset', recursive=False, properties=['name']):
# returns a dict with the following structure:
#   dataset:
#   ---> properties:
#        ---> list of properties
#   ---> snapshot name:
#        ---> list of property values

    ordered_list={'properties': properties[1:], 'values': {}}
    if recursive: recurse="-r "
    else:
        recurse=""
    props = '-o' + ','.join(properties)
    command = (REMOTE_COMMAND, host, ZFS_COMMAND, "list", GLOBAL_PARAMETERS, props, "-t", type, recurse, dataset)
    try:
        for line in check_output(command).split('\n'):
            if line !="":
                prop_values = line.split()
                dataset, snapshot = prop_values.pop(0).split('@')
                if not ordered_list['values'].has_key(dataset):
                    ordered_list['values'][dataset] = {}
                ordered_list['values'][dataset][snapshot] = prop_values

    except CalledProcessError:
        return ordered_list
    return ordered_list

def send_dataset(host1, host2, source_set, target_path, first, last):
    command = (REMOTE_COMMAND, host1, "%s send -R %s@%s | %s %s %s %s %s recv -vF %s" % ( \
                        ZFS_COMMAND, source_set, last, REMOTE_COMMAND, REMOTE_COMMAND_PARAMS,
                        REMOTE_COMMAND_PARAMS2, host2, ZFS_COMMAND, target_path))
    print "sending dataset %s from %s to %s" % (source_set, first, last)
    #keep_command = (REMOTE_COMMAND, host2, ZFS_COMMAND, 'hold', 'keep', snapshot)
    try:
        for line in check_output(command).split('\n'):
            if line != "":
                print line
#        for line in check_output(keep_command).split('\n'):
#            if line != "":
#                print line
    except CalledProcessError:
        print "error sending dataset %s" % source_set
        print "command = %s" % str(command)
        return "send dataset failed"
    return True



def send_snapshot(host1, host2, source_set, target_path, dataset, snapshots):
    command = (REMOTE_COMMAND, host1, "%s send -I %s %s@%s | %s %s %s %s %s recv -vF %s" % \
                        (ZFS_COMMAND, snapshots[0], dataset, snapshots[-1], REMOTE_COMMAND, REMOTE_COMMAND_PARAMS,
                            REMOTE_COMMAND_PARAMS2, host2, ZFS_COMMAND,
                            target_path))
    try:
        print "sending snapshots %s to %s@%s" % (snapshots[0], dataset, snapshots[-1])
        for line in check_output(command).split('\n'):
            print line
    except CalledProcessError:
        print "failure sending snapshots"<commit_msg>bold (with *) for email output when sent through cron job<commit_after># Zfs helper functions

from subprocess import check_output, CalledProcessError

ZFS_COMMAND="zfs"
REMOTE_COMMAND="ssh"
REMOTE_COMMAND_PARAMS="-c"
REMOTE_COMMAND_PARAMS2="arcfour,blowfish-cbc"
GLOBAL_PARAMETERS="-H"
DRY="echo"

# this function aims to get a list of a remote dataset
def list(host, dataset, type='dataset', recursive=False, properties=['name']):
# returns a dict with the following structure:
#   dataset:
#   ---> properties:
#        ---> list of properties
#   ---> snapshot name:
#        ---> list of property values

    ordered_list={'properties': properties[1:], 'values': {}}
    if recursive: recurse="-r "
    else:
        recurse=""
    props = '-o' + ','.join(properties)
    command = (REMOTE_COMMAND, host, ZFS_COMMAND, "list", GLOBAL_PARAMETERS, props, "-t", type, recurse, dataset)
    try:
        for line in check_output(command).split('\n'):
            if line !="":
                prop_values = line.split()
                dataset, snapshot = prop_values.pop(0).split('@')
                if not ordered_list['values'].has_key(dataset):
                    ordered_list['values'][dataset] = {}
                ordered_list['values'][dataset][snapshot] = prop_values

    except CalledProcessError:
        return ordered_list
    return ordered_list

def send_dataset(host1, host2, source_set, target_path, first, last):
    command = (REMOTE_COMMAND, host1, "%s send -R %s@%s | %s %s %s %s %s recv -vF %s" % ( \
                        ZFS_COMMAND, source_set, last, REMOTE_COMMAND, REMOTE_COMMAND_PARAMS,
                        REMOTE_COMMAND_PARAMS2, host2, ZFS_COMMAND, target_path))
    print "sending dataset %s from %s to %s" % (source_set, first, last)
    #keep_command = (REMOTE_COMMAND, host2, ZFS_COMMAND, 'hold', 'keep', snapshot)
    try:
        for line in check_output(command).split('\n'):
            if line != "":
                print line
#        for line in check_output(keep_command).split('\n'):
#            if line != "":
#                print line
    except CalledProcessError:
        print "*error sending dataset %s*" % source_set
        print "command = %s" % str(command)
        return "send dataset *failed*"
    return True



def send_snapshot(host1, host2, source_set, target_path, dataset, snapshots):
    command = (REMOTE_COMMAND, host1, "%s send -I %s %s@%s | %s %s %s %s %s recv -vF %s" % \
                        (ZFS_COMMAND, snapshots[0], dataset, snapshots[-1], REMOTE_COMMAND, REMOTE_COMMAND_PARAMS,
                            REMOTE_COMMAND_PARAMS2, host2, ZFS_COMMAND,
                            target_path))
    try:
        print "sending snapshots %s to %s@%s" % (snapshots[0], dataset, snapshots[-1])
        for line in check_output(command).split('\n'):
            print line
    except CalledProcessError:
        print "*failure* sending snapshots"
{'fraction_non_alphanumeric': 0.1268502581755594, 'fraction_numerical': 0.006196213425129088, 'mean_word_length': 2.647834274952919, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1583764', 'n_tokens_mistral': 1835, 'n_tokens_neox': 1805, 'n_words': 478}