<commit_before>#!/bin/sh

#
# This script is used to build a stub universe for Kafka, which may be used to install unreleased code via dcos-cli.
#
# When running this script, the user must provide a base path for the location of the build artifacts.
# For example, if the packages were stored in https://aws-hello-2.com/path/to/kafka, then that should be the arg.
#

DIST_PATH="$(dirname $0)"
INPUT_DIR="${DIST_PATH}/stub-universe"
OUTPUT_DIR="${DIST_PATH}/build"
OUTPUT_DIR_CONTENT="${OUTPUT_DIR}/stub-universe"
OUTPUT_RESOURCE_JSON="${OUTPUT_DIR_CONTENT}/repo/packages/K/kafka/2/resource.json"
RESOURCE_JSON_SEARCH="{{artifact-dir}}"
OUTPUT_ZIP="stub-universe.zip"

# Validate path protocol
case "$1" in
    "")
        echo "Syntax: $0 <artifact base path>"
        echo "  Artifact base path contains container-hook-x.y.z.tgz and kafka-scheduler-x.y.z-uber.jar"
        echo "  Example (HTTP): $ $0 http://example.com/path/to/artifacts/"
        echo "  Example (files): $ $0 file:///path/to/artifacts/"
        exit 1
        ;;
    http://*)
        ;; # OK!
    https://*)
        ;; # OK!
    file://*)
        ;; # OK!
    *)
        echo "Unsupported protocol (expects http://, https://, or file://): $1"
        exit 1
        ;;
esac

# Normalize path: Remove / if present
case "$1" in
    */)
        ARTIFACT_PATH="${1%?}"
        ;;
    *)
        ARTIFACT_PATH="$1"
        ;;
esac

# Initialize output dir, remove any preexisting artifacts
mkdir -p $(dirname ${OUTPUT_DIR})
if [ $? -ne 0 ]; then
   echo "output mkdir failed, exiting"
   exit 1
fi

rm -rf ${OUTPUT_DIR_CONTENT}
if [ $? -ne 0 ]; then
   echo "output cleanup rm failed, exiting"
   exit 1
fi

cp -av ${INPUT_DIR} ${OUTPUT_DIR}
if [ $? -ne 0 ]; then
   echo "input->output copy failed, exiting"
   exit 1
fi

# Update urls in resource.json. Use commas as delimiter in sed
echo "Inserting base path '${ARTIFACT_PATH}' into ${OUTPUT_RESOURCE_JSON}"
sed -i "s,${RESOURCE_JSON_SEARCH},${ARTIFACT_PATH},g" ${OUTPUT_RESOURCE_JSON}
if [ $? -ne 0 ]; then
   echo "sed failed, exiting"
   exit 1
fi
echo "Edited version:"
cat ${OUTPUT_RESOURCE_JSON}

# Create artifact
OLD_PWD=pwd
cd ${OUTPUT_DIR}

rm -f ${OUTPUT_ZIP}
if [ $? -ne 0 ]; then
    echo "zip cleanup rm failed, exiting"
    exit 1
fi

zip -r ${OUTPUT_ZIP} $(basename ${OUTPUT_DIR_CONTENT})
if [ $? -ne 0 ]; then
    echo "zip ${OUTPUT_DIR_CONTENT} -> ${OUTPUT_ZIP} failed, exiting"
    exit 1
fi

echo "---"
echo "Built Stub Universe package: ${OUTPUT_DIR}/${OUTPUT_ZIP}"
echo "---"
<commit_msg>Hack to work around missing 'zip' command on some systems<commit_after>#!/bin/sh

#
# This script is used to build a stub universe for Kafka, which may be used to install unreleased code via dcos-cli.
#
# When running this script, the user must provide a base path for the location of the build artifacts.
# For example, if the packages were stored in https://aws-hello-2.com/path/to/kafka, then that should be the arg.
#

DIST_PATH="$(dirname $0)"
INPUT_DIR="${DIST_PATH}/stub-universe"
OUTPUT_DIR="${DIST_PATH}/build"
OUTPUT_DIR_CONTENT="${OUTPUT_DIR}/stub-universe"
OUTPUT_RESOURCE_JSON="${OUTPUT_DIR_CONTENT}/repo/packages/K/kafka/2/resource.json"
RESOURCE_JSON_SEARCH="{{artifact-dir}}"
OUTPUT_ZIP="stub-universe.zip"

# Validate path protocol
case "$1" in
    "")
        echo "Syntax: $0 <artifact base path>"
        echo "  Artifact base path contains container-hook-x.y.z.tgz and kafka-scheduler-x.y.z-uber.jar"
        echo "  Example (HTTP): $ $0 http://example.com/path/to/artifacts/"
        echo "  Example (files): $ $0 file:///path/to/artifacts/"
        exit 1
        ;;
    http://*)
        ;; # OK!
    https://*)
        ;; # OK!
    file://*)
        ;; # OK!
    *)
        echo "Unsupported protocol (expects http://, https://, or file://): $1"
        exit 1
        ;;
esac

# Normalize path: Remove / if present
case "$1" in
    */)
        ARTIFACT_PATH="${1%?}"
        ;;
    *)
        ARTIFACT_PATH="$1"
        ;;
esac

# Initialize output dir, remove any preexisting artifacts
mkdir -p $(dirname ${OUTPUT_DIR})
if [ $? -ne 0 ]; then
   echo "output mkdir failed, exiting"
   exit 1
fi

rm -rf ${OUTPUT_DIR_CONTENT}
if [ $? -ne 0 ]; then
   echo "output cleanup rm failed, exiting"
   exit 1
fi

cp -av ${INPUT_DIR} ${OUTPUT_DIR}
if [ $? -ne 0 ]; then
   echo "input->output copy failed, exiting"
   exit 1
fi

# Update urls in resource.json. Use commas as delimiter in sed
echo "Inserting base path '${ARTIFACT_PATH}' into ${OUTPUT_RESOURCE_JSON}"
sed -i "s,${RESOURCE_JSON_SEARCH},${ARTIFACT_PATH},g" ${OUTPUT_RESOURCE_JSON}
if [ $? -ne 0 ]; then
   echo "sed failed, exiting"
   exit 1
fi
echo "Edited version:"
cat ${OUTPUT_RESOURCE_JSON}

# Create artifact
OLD_PWD=pwd
cd ${OUTPUT_DIR}

rm -f ${OUTPUT_ZIP}
if [ $? -ne 0 ]; then
    echo "zip cleanup rm failed, exiting"
    exit 1
fi

ZIPCMD=$(which zipaa)
if [ "${ZIPCMD}" = "" ]; then
    echo "'zip' command not available, falling back to python hack..."
    python -c "import os, os.path, zipfile

zipout = zipfile.ZipFile(\"${OUTPUT_ZIP}\", 'w', zipfile.ZIP_DEFLATED)
for root, dirs, files in os.walk(\"$(basename ${OUTPUT_DIR_CONTENT})\"):
  for f in files:
    path = os.path.join(root, f)
    print(\"adding: %s\" % path)
    zipout.write(path)"
else
    ${ZIPCMD} -r ${OUTPUT_ZIP} $(basename ${OUTPUT_DIR_CONTENT})
fi
if [ $? -ne 0 ]; then
    echo "zip ${OUTPUT_DIR_CONTENT} -> ${OUTPUT_ZIP} failed, exiting"
    exit 1
fi

echo "---"
echo "Built Stub Universe package: ${OUTPUT_DIR}/${OUTPUT_ZIP}"
echo "---"

{'fraction_non_alphanumeric': 0.15069482042952537, 'fraction_numerical': 0.009023641941887746, 'mean_word_length': 3.1420029895366217, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '212462', 'n_tokens_mistral': 1992, 'n_tokens_neox': 1913, 'n_words': 580}