<commit_before>#!/bin/sh
#
#     ___   ___                       .___
#    /   \  \  | - L3 for OpenStack - | _/
#   /  _  \ |  | _______    ____    __| | ____
#  /  /_\  \|  |/ /\__  \  /    \  / __ |\__  \
# /    |    \    <  / __ \|   |  \/ /_/ | / __ \_
# \____|__  /__|_ \(____  /___|  /\____ |(____  /
#         \/     \/     \/     \/      \/     \/
#
# This script creates an Akanda Live CD - powered by OpenBSD, Python, and
# Flask - and # lets you customize it.
#
# Copyright (c) 2009 <NAME>. Released under the BSD license.
# Copyright (c) 2012 New Dream Network, LLC (DreamHost).
#
# First release 2009-06-20
# Akanda release 2012-10-14
#
# Notes:
#
# * Modified 2012 by DreamHost <<EMAIL>> for use with
#   Akanda

###############################################################################
# Defaults
###############################################################################
MAJ=5                    # Version major number
MIN=1                    # Version minor number
ARCH=amd64                # Architecture
TZ=UTC                   # Time zones are in /usr/share/zoneinfo
# The base sets that should be installed on the akanda live cd
SETS="base etc man"
# Additional packages that should be installed on the akanda live cd
PACKAGES="ntp python-2.7.1p12 py-pip wget"


WDIR=/usr/local/akanda-livecdx            # Working directory
CDBOOTDIR=$WDIR/$MAJ.$MIN/$ARCH        # CD Boot directory
OUTDIR=/tmp
HERE=`pwd`

# Mirror to use to download the OpenBSD files
#BASEURL=http://ftp-stud.fht-esslingen.de/pub/OpenBSD
BASEURL=http://openbsd.mirrors.pair.com
MIRROR=$BASEURL/$MAJ.$MIN/$ARCH
PKG_PATH=$BASEURL/$MAJ.$MIN/packages/$ARCH
DNS=8.8.8.8            # Google DNS Server to use in live cd (change accordingly)


CLEANUP=no                    # Clean up downloaded files and workdir (disabled by default)

# End of user configuration
###############################################################################

# global variables

SCRIPTNAME=$(basename $0 .sh)

EXIT_SUCCESS=0
EXIT_FAILED=1
EXIT_ERROR=2
EXIT_BUG=10

VERSION="1.0.0"

# base functions

# In case of an error it is wise to show the correct usage of the script.
function usage {
    echo >&2
    echo -e "Usage: $SCRIPTNAME \t[-A <arch>] [-h] [-M <major>] [-m <minor>] [-P <packages>]" >&2
    echo -e "                   \t\t[-S <sets>] [-T <timezone>] [-V] [-W <workdir>] [-U <url>]" >&2
    echo >&2
    echo "This program creates an OpenBSD live cd and lets you customize it." >&2
    echo "The software is released under BSD license. Use it at your own risk!" >&2
    echo "Copyright (c) 2009 <NAME>. Email: reiner[AT]rottmann.it" >&2
    echo "Copyright (c) 2012 New Dream Network, LLC. Email: dev-community[AT]dreamhost.com" >&2
    echo >&2
    echo -e "  -A :\tselect architecture (default: $ARCH)" >&2
    echo -e "  -h :\tgive this help list" >&2
    echo -e "  -M :\tselect OpenBSD major version (default: $MAJ)" >&2
    echo -e "  -m :\tselect OpenBSD minor version (default: $MIN)" >&2
    echo -e "  -P :\tselect additional packages to install" >&2
    echo -e "      \t(default: $PACKAGES)" >&2
    echo -e "  -S :\tselect base sets (default: $SETS)" >&2
    echo -e "  -T :\tselect timezone (default: $TZ)" >&2
    echo -e "  -U :\tselect url of nearest OpenBSD mirror (default: $MIRROR)" >&2
    echo -e "  -u :\tselect url of nearest OpenBSD from mirror list (requires wget)" >&2
    echo -e "  -V :\tprint version" >&2
    echo -e "  -W :\tselect working directory (default: $WDIR)" >&2
    echo >&2
    echo -e "Example:" >&2
    echo -e "# $SCRIPTNAME -A amd64 -M 4 -m 5 -W /tmp/livecd" >&2
    echo >&2
    [[ $# -eq 1 ]] && exit $1 || exit $EXIT_FAILED
}

# own functions
# This function lets the user choose an OpenBSD mirror
function choosemirror {
    req="wget"
    for i in $req
    do
        if ! which $i >/dev/null; then
            echo "Missing $i. Exiting."
            exit $EXIT_ERROR
        fi
    done

    mirrorlist=$(wget -q -O - http://www.openbsd.org/ftp.html#ftp | sed -n 's#<a href=\"\(ftp://.*\)/">#\1#p'|sort)

    echo "Please select mirror from the list below:"

    mirr=""
    while [ -z "$mirr" ] ; do
            m=1
            for i in $mirrorlist
            do
                    echo $m. "$i"
                    m=$(($m+1))
            done
            echo -n "Your choice? : "
            read choice
            mirr=$(echo "$mirrorlist" | sed -n $choice,${choice}p| sed s#^\ *##g)
    done
    BASEURL=$mirr
    MIRROR=$BASEURL/$MAJ.$MIN/$ARCH
    PKG_PATH=$BASEURL/$MAJ.$MIN/packages/$ARCH
    CDBOOTDIR=$WDIR/$MAJ.$MIN/$ARCH
}

# This function may be used for cleanup before ending the program
function cleanup {
    echo
}

function makedeps {
    echo "[*] Installing dependencies for make"
    pkg_add -i bison
    pkg_add -i m4
    pkg_add -i gmake
}


# This is the main function that creates the OpenBSD livecd
function livecd {
    echo "[*] Akanda (powered by OpenBSD) LiveCD script"
    echo "[*] The software is released under BSD license. Use it at your own risk!" >&2
    echo "[*] Copyright (c) 2009 <NAME>." >&2
    echo "[*] Copyright (c) 2012 New Dream Network, LLC (DreamHost)." >&2
    echo "[*] This script is released under the BSD License."
    uname -a | grep OpenBSD || echo "[*] WARNING: This software should run on an OpenBSD System!"
    date
    echo "[*] Setting up the build environment..."
    mkdir -p $WDIR

     if [[ $CHMIRROR = y ]] ; then
        echo "[*] Selecting OpenBSD mirror..."
        choosemirror
        echo $MIRROR
    fi

    # Create CD Boot directory
    mkdir -p $CDBOOTDIR && cd $CDBOOTDIR

    echo "[*] Downloading files needed for CD Boot..."
    CDBOOTFILES="cdbr cdboot bsd"
    cd $CDBOOTDIR && for i in $CDBOOTFILES; do test -f $CDBOOTDIR/$i || ftp -o $CDBOOTDIR/$i -m $MIRROR/$i; done


    echo "[*] Downloading file sets ($SETS)..."
    cd $WDIR && for i in $SETS; do test -f $WDIR/$i$MAJ$MIN.tgz || ftp -o $WDIR/$i$MAJ$MIN.tgz -m $MIRROR/$i$MAJ$MIN.tgz; done

    echo "[*] Extracting file sets ($SETS)..."
    cd $WDIR && for i in $SETS; do tar xzpf $WDIR/$i$MAJ$MIN.tgz; done

    if [ $CLEANUP="yes" ];then<commit_msg>Be more careful about download failures<commit_after>#!/bin/sh
#
#     ___   ___                       .___
#    /   \  \  | - L3 for OpenStack - | _/
#   /  _  \ |  | _______    ____    __| | ____
#  /  /_\  \|  |/ /\__  \  /    \  / __ |\__  \
# /    |    \    <  / __ \|   |  \/ /_/ | / __ \_
# \____|__  /__|_ \(____  /___|  /\____ |(____  /
#         \/     \/     \/     \/      \/     \/
#
# This script creates an Akanda Live CD - powered by OpenBSD, Python, and
# Flask - and # lets you customize it.
#
# Copyright (c) 2009 <NAME>. Released under the BSD license.
# Copyright (c) 2012 New Dream Network, LLC (DreamHost).
#
# First release 2009-06-20
# Akanda release 2012-10-14
#
# Notes:
#
# * Modified 2012 by DreamHost <<EMAIL>> for use with
#   Akanda

###############################################################################
# Defaults
###############################################################################
MAJ=5                    # Version major number
MIN=1                    # Version minor number
ARCH=amd64                # Architecture
TZ=UTC                   # Time zones are in /usr/share/zoneinfo
# The base sets that should be installed on the akanda live cd
SETS="base etc man"
# Additional packages that should be installed on the akanda live cd
PACKAGES="ntp python-2.7.1p12 py-pip wget"


WDIR=/usr/local/akanda-livecdx            # Working directory
CDBOOTDIR=$WDIR/$MAJ.$MIN/$ARCH        # CD Boot directory
OUTDIR=/tmp
HERE=`pwd`

# Mirror to use to download the OpenBSD files
#BASEURL=http://ftp-stud.fht-esslingen.de/pub/OpenBSD
#BASEURL=http://openbsd.mirrors.pair.com
BASEURL=ftp://ftp3.usa.openbsd.org/pub/OpenBSD
MIRROR=$BASEURL/$MAJ.$MIN/$ARCH
PKG_PATH=$BASEURL/$MAJ.$MIN/packages/$ARCH
DNS=8.8.8.8            # Google DNS Server to use in live cd (change accordingly)


#CLEANUP=no                    # Clean up downloaded files and workdir (disabled by default)
CLEANUP=yes

# End of user configuration
###############################################################################

# global variables

SCRIPTNAME=$(basename $0 .sh)

EXIT_SUCCESS=0
EXIT_FAILED=1
EXIT_ERROR=2
EXIT_BUG=10

VERSION="1.0.0"

# base functions

# In case of an error it is wise to show the correct usage of the script.
function usage {
    echo >&2
    echo -e "Usage: $SCRIPTNAME \t[-A <arch>] [-h] [-M <major>] [-m <minor>] [-P <packages>]" >&2
    echo -e "                   \t\t[-S <sets>] [-T <timezone>] [-V] [-W <workdir>] [-U <url>]" >&2
    echo >&2
    echo "This program creates an OpenBSD live cd and lets you customize it." >&2
    echo "The software is released under BSD license. Use it at your own risk!" >&2
    echo "Copyright (c) 2009 <NAME>. Email: reiner[AT]rottmann.it" >&2
    echo "Copyright (c) 2012 New Dream Network, LLC. Email: dev-community[AT]dreamhost.com" >&2
    echo >&2
    echo -e "  -A :\tselect architecture (default: $ARCH)" >&2
    echo -e "  -h :\tgive this help list" >&2
    echo -e "  -M :\tselect OpenBSD major version (default: $MAJ)" >&2
    echo -e "  -m :\tselect OpenBSD minor version (default: $MIN)" >&2
    echo -e "  -P :\tselect additional packages to install" >&2
    echo -e "      \t(default: $PACKAGES)" >&2
    echo -e "  -S :\tselect base sets (default: $SETS)" >&2
    echo -e "  -T :\tselect timezone (default: $TZ)" >&2
    echo -e "  -U :\tselect url of nearest OpenBSD mirror (default: $MIRROR)" >&2
    echo -e "  -u :\tselect url of nearest OpenBSD from mirror list (requires wget)" >&2
    echo -e "  -V :\tprint version" >&2
    echo -e "  -W :\tselect working directory (default: $WDIR)" >&2
    echo >&2
    echo -e "Example:" >&2
    echo -e "# $SCRIPTNAME -A amd64 -M 4 -m 5 -W /tmp/livecd" >&2
    echo >&2
    [[ $# -eq 1 ]] && exit $1 || exit $EXIT_FAILED
}

# own functions
# This function lets the user choose an OpenBSD mirror
function choosemirror {
    req="wget"
    for i in $req
    do
        if ! which $i >/dev/null; then
            echo "Missing $i. Exiting."
            exit $EXIT_ERROR
        fi
    done

    mirrorlist=$(wget -q -O - http://www.openbsd.org/ftp.html#ftp | sed -n 's#<a href=\"\(ftp://.*\)/">#\1#p'|sort)

    echo "Please select mirror from the list below:"

    mirr=""
    while [ -z "$mirr" ] ; do
            m=1
            for i in $mirrorlist
            do
                    echo $m. "$i"
                    m=$(($m+1))
            done
            echo -n "Your choice? : "
            read choice
            mirr=$(echo "$mirrorlist" | sed -n $choice,${choice}p| sed s#^\ *##g)
    done
    BASEURL=$mirr
    MIRROR=$BASEURL/$MAJ.$MIN/$ARCH
    PKG_PATH=$BASEURL/$MAJ.$MIN/packages/$ARCH
    CDBOOTDIR=$WDIR/$MAJ.$MIN/$ARCH
}

# This function may be used for cleanup before ending the program
function cleanup {
    echo
}

function makedeps {
    echo "[*] Installing dependencies for make"
    pkg_add -i bison
    pkg_add -i m4
    pkg_add -i gmake
}


# This is the main function that creates the OpenBSD livecd
function livecd {
    echo "[*] Akanda (powered by OpenBSD) LiveCD script"
    echo "[*] The software is released under BSD license. Use it at your own risk!" >&2
    echo "[*] Copyright (c) 2009 <NAME>." >&2
    echo "[*] Copyright (c) 2012 New Dream Network, LLC (DreamHost)." >&2
    echo "[*] This script is released under the BSD License."
    uname -a | grep OpenBSD || echo "[*] WARNING: This software should run on an OpenBSD System!"
    date
    echo "[*] Setting up the build environment..."
    mkdir -p $WDIR

     if [[ $CHMIRROR = y ]] ; then
        echo "[*] Selecting OpenBSD mirror..."
        choosemirror
        echo $MIRROR
    fi

    # Create CD Boot directory
    mkdir -p $CDBOOTDIR && cd $CDBOOTDIR

    echo "[*] Downloading files needed for CD Boot..."
    CDBOOTFILES="cdbr cdboot bsd"
    cd $CDBOOTDIR && for i in $CDBOOTFILES; do test -f $CDBOOTDIR/$i || ftp -o $CDBOOTDIR/$i -m $MIRROR/$i; done
    typeset missing=""
    cd $CDBOOTDIR && for i in $CDBOOTFILES; do test -f $CDBOOTDIR/$i || missing="$missing $i"; done
    if [ ! -z "$missing" ]
    then
        echo "Missing download files: $missing" 1>&2
        exit 1
    fi

    echo "[*] Downloading file sets ($SETS)..."
    cd $WDIR && for i in $SETS; do test -f $WDIR/$i$MAJ$MIN.tgz || ftp -o $WDIR/$i$MAJ$MIN.tgz -m $MIRROR/$i$MAJ$MIN.tgz; done
    typeset missing=""
    cd $WDIR && for i in $SETS; do test -f $WDIR/$i$MAJ$MIN.tgz || missing="$missing $i$MAJ$MIN.tgz"; done
    if [ ! -z "$missing" ]
    then
        echo "Missing download file sets: $missing" 1>&2
        exit 1
    fi

    echo "[*] Extracting file sets ($SETS)..."
    cd $WDIR && for i in $SETS; do tar xzpf $WDIR/$i$MAJ$MIN.tgz; done

    if [ $CLEANUP="yes" ];then
{'fraction_non_alphanumeric': 0.17639288211982282, 'fraction_numerical': 0.01686222705726941, 'mean_word_length': 2.3664661260789956, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 91, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6321566', 'n_tokens_mistral': 4676, 'n_tokens_neox': 4356, 'n_words': 1413}