<commit_before>#!/bin/bash
#
# SSH to datasets.joyent.com and push all datasets to images.joyent.com.
#
# WARNING: Right now, at least, this should only be used for dev.
# It might eventually be useful for keeping datasets.jo and images.jo in sync.
#

if [ "$TRACE" != "" ]; then
    export PS4='[\D{%FT%TZ}] ${BASH_SOURCE}:${LINENO}: ${FUNCNAME[0]:+${FUNCNAME[0]}(): }'
    set -o xtrace
fi
set -o errexit
set -o pipefail


TOP=$(unset CDPATH; cd $(dirname $0)/; pwd)
SSH_OPTIONS="-q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
SSH="ssh $SSH_OPTIONS"
DATASETS_LOGIN=root@datasets



#---- mainline

echo '# Push datasets.joyent.com datasets to images.joyent.com.'
echo '# WARNING: images.joyent.com is a *production* server.'
echo '# Press <Enter> to continue, <Ctrl+C> to cancel.'
read

$SSH -T $DATASETS_LOGIN <<SCRIPT

# TODO: should be images.joyent.com when that is DNS'd.
IMGAPI_URL=https://172.16.17.32

function imgapi {
    local path=\$1
    shift
    curl -k --connect-timeout 10 -sS -i -H accept:application/json \
        --url \$IMGAPI_URL\$path "\$@" | json -q
}

function push2images {
    local manifests=\$(ls -1 /shared/dsapi/manifests/*.dsmanifest)
    for manifest in \$manifests; do
        local uuid=\$(json uuid < \$manifest)
        local name=\$(json name < \$manifest)
        local version=\$(json version < \$manifest)
        local restricted_to_uuid=\$(json restricted_to_uuid < \$manifest)
        local type_=\$(json type < \$manifest)
        if [[ "\$type_" == "vmimage" ]]; then
            echo "Skipping import of image \$uuid: vmimage type is invalid."
            continue
        elif [[ "\$type_" == "zvol" ]]; then
            echo "Skipping import of image \$uuid: zvol not quite supported yet."
            continue
        elif [[ -n "\$restricted_to_uuid" ]]; then
            echo "Skipping import of image \$uuid: private."
            continue
        fi
        local status=\$(imgapi /images/\$uuid | head -1 | awk '{print \$2}')
        if [[ "\$status" == "404" ]]; then
            local file=\$(ls /shared/dsapi/assets/\$uuid/*)
            echo "Importing image \$uuid \$name-\$version into IMGAPI."
            echo "  manifest: \$manifest"
            echo "  file:     \$file"
            [[ -f "\$file" ]] || fatal "Image \$uuid file '\$file' not found."
            imgapi /images/\$uuid?action=import -d @\$manifest -f
            imgapi /images/\$uuid/file -T \$file -f
            imgapi /images/\$uuid?action=activate -X POST -f
        elif [[ "\$status" == "200" ]]; then
            echo "Skipping import of image \$uuid: already in IMGAPI."
        else
            echo "Error checking if image \$uuid is in IMGAPI: HTTP \$status"
        fi
    done
}

push2images

SCRIPT<commit_msg>IMGAPI-62: improve datasets2images importer to use 'joyent-imgadm import' command<commit_after>#!/bin/bash
#
# SSH to datasets.joyent.com and push all datasets to images.joyent.com.
#
# WARNING: Right now, at least, this should only be used for dev.
# It might eventually be useful for keeping datasets.jo and images.jo in sync.
#

if [ "$TRACE" != "" ]; then
    export PS4='[\D{%FT%TZ}] ${BASH_SOURCE}:${LINENO}: ${FUNCNAME[0]:+${FUNCNAME[0]}(): }'
    set -o xtrace
fi
set -o errexit
set -o pipefail


TOP=$(unset CDPATH; cd $(dirname $0)/; pwd)
SSH_OPTIONS="-q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
SSH="ssh $SSH_OPTIONS"
DATASETS_LOGIN=root@datasets



#---- mainline

echo '# Push datasets.joyent.com datasets to images.joyent.com.'
echo '# WARNING: images.joyent.com is a *production* server.'
echo '# Press <Enter> to continue, <Ctrl+C> to cancel.'
read

$SSH -A -T $DATASETS_LOGIN <<SCRIPT

if [ "$TRACE" != "" ]; then
    #export PS4='[\D{%FT%TZ}] \${BASH_SOURCE}:\${LINENO}: \${FUNCNAME[0]:+\${FUNCNAME[0]}(): }'
    set -o xtrace
fi
set -o errexit
set -o pipefail

export JOYENT_IMGADM_IDENTITY=b3:f0:a1:6c:18:3b:47:63:ae:6e:57:22:74:71:d4:bc
export JOYENT_IMGADM_USER=trentm
JOYENT_IMGADM=\$HOME/bin/joyent-imgadm
JSON=\$HOME/bin/json


function push2images {
    local have_uuids=\$(\$JOYENT_IMGADM list -a -j | \$JSON -a uuid)
    local manifests=\$(ls -1 /shared/dsapi/manifests/*.dsmanifest)
    for manifest in \$manifests; do
        local uuid=\$(\$JSON uuid < \$manifest)
        local name=\$(\$JSON name < \$manifest)
        local version=\$(\$JSON version < \$manifest)
        local restricted_to_uuid=\$(\$JSON restricted_to_uuid < \$manifest)
        local type_=\$(\$JSON type < \$manifest)
        if [[ "\$type_" == "vmimage" ]]; then
            echo "Skipping import of image \$uuid: vmimage type is invalid."
            continue
        elif [[ "\$type_" == "zvol" ]]; then
            echo "Skipping import of image \$uuid: zvol not quite supported yet."
            continue
        elif [[ -n "\$restricted_to_uuid" ]]; then
            echo "Skipping import of image \$uuid: private."
            continue
        fi
        if [[ -z "\$(echo "\$have_uuids" | grep \$uuid)" ]]; then
            local file=\$(ls /shared/dsapi/assets/\$uuid/*)
            echo "Importing image \$uuid \$name-\$version into IMGAPI."
            echo "  manifest: \$manifest"
            echo "  file:     \$file"
            [[ -f "\$file" ]] || fatal "Image \$uuid file '\$file' not found."
            \$JOYENT_IMGADM import -P -m "\$manifest" -f "\$file"
        else
            echo "Skipping import of image \$uuid: already in IMGAPI."
        fi
    done
}

push2images

SCRIPT
{'fraction_non_alphanumeric': 0.15809558422678538, 'fraction_numerical': 0.011084862802107942, 'mean_word_length': 2.5463917525773194, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 7, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5291716', 'n_tokens_mistral': 1935, 'n_tokens_neox': 1826, 'n_words': 533}