<commit_before>#!/bin/bash

set -e

export IMAGE_SIZE=$DIB_IMAGE_SIZE
# This will unset parameter DIB_IMAGE_SIZE for Ubuntu and Fedora vanilla images
unset DIB_IMAGE_SIZE

# default debug setting should be false
IMAGE_GENERATION_DEBUG_MODE="false"

while getopts "p:i:v:d:" opt; do
  case $opt in
    p)
      PLUGIN=$OPTARG
    ;;
    i)
      IMAGE_TYPE=$OPTARG
     ;;
    v)
      HADOOP_VERSION=$OPTARG
    ;;
    d)
      IMAGE_GENERATION_DEBUG_MODE=$OPTARG
    ;;
    *)
      echo
      echo "Usage: $(basename $0)"
      echo "         [-p vanilla|spark|hdp|idh]"
      echo "         [-i ubuntu|fedora|centos]"
      echo "         [-v 1|2|plain]"
      echo "         [-d true|false]"
      echo "   '-p' is plugin version (default: vanilla)"
      echo "   '-i' is image type (default: all supported by plugin)"
      echo "   '-v' is hadoop version (default: all supported by plugin)"
      echo "   '-d' controls the debug mode for image generation (false by default)"
      echo
      echo "You shouldn't specify hadoop version and image type for spark plugin"
      echo "You shouldn't specify image type for hdp plugin"
      echo "Version 'plain' could be specified for hdp plugin only"
      echo "Debug mode should only be enabled for local debugging purposes, not for production systems"
      echo "By default all images for all plugins will be created"
      echo
      exit 1
    ;;
  esac
done


# Checks of input
if [ -n "$PLUGIN" -a "$PLUGIN" != "vanilla" -a "$PLUGIN" != "spark" -a "$PLUGIN" != "hdp" -a "$PLUGIN" != "idh" ]; then
  echo -e "Unknown plugin selected.\nAborting"
  exit 1
fi

if [ -n "$IMAGE_TYPE" -a "$IMAGE_TYPE" != "ubuntu" -a "$IMAGE_TYPE" != "fedora" -a "$IMAGE_TYPE" != "centos" ]; then
  echo -e "Unknown image type selected.\nAborting"
  exit 1
fi

if [ -n "$HADOOP_VERSION" -a "$HADOOP_VERSION" != "1" -a "$HADOOP_VERSION" != "2" -a "$HADOOP_VERSION" != "plain" ]; then
  echo -e "Unknown hadoop version selected.\nAborting"
  exit 1
fi

if [ -n "$IMAGE_GENERATION_DEBUG_MODE" -a "$IMAGE_GENERATION_DEBUG_MODE" != "true" -a "$IMAGE_GENERATION_DEBUG_MODE" != "false" ]; then
  echo -e "Unknown debug mode.\nAborting"
  exit 1
else if [ -z "$IMAGE_GENERATION_DEBUG_MODE"  ]; then
     echo -e "Empty string passed for debug mode.  Invalid input. \nAborting"
     exit 1
     fi
fi

if [ "$PLUGIN" = "vanilla" -a "$HADOOP_VERSION" = "plain" ]; then
  echo "Impossible combination.\nAborting"
  exit 1
fi

if [ "$PLUGIN" = "idh" -a "$HADOOP_VERSION" = "plain" ]; then
  echo "Impossible combination.\nAborting"
  exit 1
fi
#################

if [ -e /etc/os-release ]; then
  platform=$(head -1 /etc/os-release)
  if [ $platform = 'NAME="Ubuntu"' ]; then
    apt-get update -y
    apt-get install qemu kpartx git -y
  elif [ $platform = 'NAME=Fedora' ]; then
    yum update -y
    yum install qemu kpartx git -y
  fi
else
  platform=$(head -1 /etc/system-release | grep -e CentOS -e 'Red Hat Enterprise Linux' || :)
  if [ -n "$platform" ]; then
    yum update -y
    yum install qemu-kvm qemu-img kpartx git -y
  else
    echo -e "Unknown Host OS. Impossible to build images.\nAborting"
    exit 2
  fi
fi

base_dir="$(dirname $(readlink -e $0))"

TEMP=$(mktemp -d diskimage-create.XXXXXX)
pushd $TEMP

export DIB_IMAGE_CACHE=$TEMP/.cache-image-create

# Working with repositories
# disk-image-builder repo

if [ -z $DIB_REPO_PATH ]; then
  git clone https://git.openstack.org/openstack/diskimage-builder
  DIB_REPO_PATH="$(pwd)/diskimage-builder"
fi

export PATH=$PATH:$DIB_REPO_PATH/bin

pushd $DIB_REPO_PATH
export DIB_COMMIT_ID=`git rev-parse HEAD`
popd

export ELEMENTS_PATH="$DIB_REPO_PATH/elements"

# sahara-image-elements repo

if [ -z $SIM_REPO_PATH ]; then
  SIM_REPO_PATH="$(dirname $base_dir)"
  if [ $(basename $SIM_REPO_PATH) != "sahara-image-elements" ]; then
    echo "Can't find Sahara-image-elements repository. Cloning it."
    git clone https://git.openstack.org/openstack/sahara-image-elements
    SIM_REPO_PATH="$(pwd)/sahara-image-elements"
  fi
fi

ELEMENTS_PATH=$ELEMENTS_PATH:$SIM_REPO_PATH/elements

pushd $SIM_REPO_PATH
export SAHARA_ELEMENTS_COMMIT_ID=`git rev-parse HEAD`
popd

#############################
# Images for Vanilla plugin #
#############################

if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then
  export JAVA_DOWNLOAD_URL=${JAVA_DOWNLOAD_URL:-"http://download.oracle.com/otn-pub/java/jdk/7u51-b13/jdk-7u51-linux-x64.tar.gz"}
  export OOZIE_HADOOP_V1_DOWNLOAD_URL=${OOZIE_HADOOP_V1_DOWNLOAD_URL:-"http://sahara-files.mirantis.com/oozie-4.0.0.tar.gz"}
  export OOZIE_HADOOP_V2_DOWNLOAD_URL=${OOZIE_HADOOP_V2_DOWNLOAD_URL:-"http://sahara-files.mirantis.com/oozie-4.0.0-hadoop-2.3.0.tar.gz"}
  export EXTJS_DOWNLOAD_URL=${EXTJS_DOWNLOAD_URL:-"http://extjs.com/deploy/ext-2.2.zip"}
  export HIVE_VERSION=${HIVE_VERSION:-"0.11.0"}

  ubuntu_elements_sequence="base vm ubuntu hadoop oozie mysql hive"
  fedora_elements_sequence="base vm fedora hadoop oozie mysql hive disable-firewall"
  centos_elements_sequence="vm rhel hadoop oozie mysql hive redhat-lsb disable-firewall"

  # Workaround for https://bugs.launchpad.net/diskimage-builder/+bug/1204824
  # https://bugs.launchpad.net/sahara/+bug/1252684
  if [ -z "$IMAGE_TYPE" -o "$IMAGE_TYPE" = "centos" -o "$IMAGE_TYPE" = "fedora" ]; then
    if [ "$platform" = 'NAME="Ubuntu"' ]; then
      echo "**************************************************************"
      echo "WARNING: As a workaround for DIB bug 1204824, you are about to"
      echo "         create a Fedora and CentOS images that has SELinux    "
      echo "         disabled. Do not use these images in production.       "
      echo "**************************************************************"
      fedora_elements_sequence="$fedora_elements_sequence selinux-permissive"
      centos_elements_sequence="$centos_elements_sequence selinux-permissive"
      suffix=".selinux-permissive"
    fi
  fi

  if [ -n "$USE_MIRRORS" ]; then
    mirror_element=" apt-mirror"
    ubuntu_elements_sequence=$ubuntu_elements_sequence$mirror_element
    mirror_element=" yum-mirror"
    fedora_elements_sequence=$fedora_elements_sequence$mirror_element
  fi
<commit_msg>Refactor logic around selinux bug 1204824<commit_after>#!/bin/bash

set -e

export IMAGE_SIZE=$DIB_IMAGE_SIZE
# This will unset parameter DIB_IMAGE_SIZE for Ubuntu and Fedora vanilla images
unset DIB_IMAGE_SIZE

# default debug setting should be false
IMAGE_GENERATION_DEBUG_MODE="false"

while getopts "p:i:v:d:" opt; do
  case $opt in
    p)
      PLUGIN=$OPTARG
    ;;
    i)
      IMAGE_TYPE=$OPTARG
     ;;
    v)
      HADOOP_VERSION=$OPTARG
    ;;
    d)
      IMAGE_GENERATION_DEBUG_MODE=$OPTARG
    ;;
    *)
      echo
      echo "Usage: $(basename $0)"
      echo "         [-p vanilla|spark|hdp|idh]"
      echo "         [-i ubuntu|fedora|centos]"
      echo "         [-v 1|2|plain]"
      echo "         [-d true|false]"
      echo "   '-p' is plugin version (default: vanilla)"
      echo "   '-i' is image type (default: all supported by plugin)"
      echo "   '-v' is hadoop version (default: all supported by plugin)"
      echo "   '-d' controls the debug mode for image generation (false by default)"
      echo
      echo "You shouldn't specify hadoop version and image type for spark plugin"
      echo "You shouldn't specify image type for hdp plugin"
      echo "Version 'plain' could be specified for hdp plugin only"
      echo "Debug mode should only be enabled for local debugging purposes, not for production systems"
      echo "By default all images for all plugins will be created"
      echo
      exit 1
    ;;
  esac
done


# Checks of input
if [ -n "$PLUGIN" -a "$PLUGIN" != "vanilla" -a "$PLUGIN" != "spark" -a "$PLUGIN" != "hdp" -a "$PLUGIN" != "idh" ]; then
  echo -e "Unknown plugin selected.\nAborting"
  exit 1
fi

if [ -n "$IMAGE_TYPE" -a "$IMAGE_TYPE" != "ubuntu" -a "$IMAGE_TYPE" != "fedora" -a "$IMAGE_TYPE" != "centos" ]; then
  echo -e "Unknown image type selected.\nAborting"
  exit 1
fi

if [ -n "$HADOOP_VERSION" -a "$HADOOP_VERSION" != "1" -a "$HADOOP_VERSION" != "2" -a "$HADOOP_VERSION" != "plain" ]; then
  echo -e "Unknown hadoop version selected.\nAborting"
  exit 1
fi

if [ -n "$IMAGE_GENERATION_DEBUG_MODE" -a "$IMAGE_GENERATION_DEBUG_MODE" != "true" -a "$IMAGE_GENERATION_DEBUG_MODE" != "false" ]; then
  echo -e "Unknown debug mode.\nAborting"
  exit 1
else if [ -z "$IMAGE_GENERATION_DEBUG_MODE"  ]; then
     echo -e "Empty string passed for debug mode.  Invalid input. \nAborting"
     exit 1
     fi
fi

if [ "$PLUGIN" = "vanilla" -a "$HADOOP_VERSION" = "plain" ]; then
  echo "Impossible combination.\nAborting"
  exit 1
fi

if [ "$PLUGIN" = "idh" -a "$HADOOP_VERSION" = "plain" ]; then
  echo "Impossible combination.\nAborting"
  exit 1
fi
#################

if [ -e /etc/os-release ]; then
  platform=$(head -1 /etc/os-release)
  if [ $platform = 'NAME="Ubuntu"' ]; then
    apt-get update -y
    apt-get install qemu kpartx git -y
  elif [ $platform = 'NAME=Fedora' ]; then
    yum update -y
    yum install qemu kpartx git -y
  fi
else
  platform=$(head -1 /etc/system-release | grep -e CentOS -e 'Red Hat Enterprise Linux' || :)
  if [ -n "$platform" ]; then
    yum update -y
    yum install qemu-kvm qemu-img kpartx git -y
  else
    echo -e "Unknown Host OS. Impossible to build images.\nAborting"
    exit 2
  fi
fi

base_dir="$(dirname $(readlink -e $0))"

TEMP=$(mktemp -d diskimage-create.XXXXXX)
pushd $TEMP

export DIB_IMAGE_CACHE=$TEMP/.cache-image-create

# Working with repositories
# disk-image-builder repo

if [ -z $DIB_REPO_PATH ]; then
  git clone https://git.openstack.org/openstack/diskimage-builder
  DIB_REPO_PATH="$(pwd)/diskimage-builder"
fi

export PATH=$PATH:$DIB_REPO_PATH/bin

pushd $DIB_REPO_PATH
export DIB_COMMIT_ID=`git rev-parse HEAD`
popd

export ELEMENTS_PATH="$DIB_REPO_PATH/elements"

# sahara-image-elements repo

if [ -z $SIM_REPO_PATH ]; then
  SIM_REPO_PATH="$(dirname $base_dir)"
  if [ $(basename $SIM_REPO_PATH) != "sahara-image-elements" ]; then
    echo "Can't find Sahara-image-elements repository. Cloning it."
    git clone https://git.openstack.org/openstack/sahara-image-elements
    SIM_REPO_PATH="$(pwd)/sahara-image-elements"
  fi
fi

ELEMENTS_PATH=$ELEMENTS_PATH:$SIM_REPO_PATH/elements

pushd $SIM_REPO_PATH
export SAHARA_ELEMENTS_COMMIT_ID=`git rev-parse HEAD`
popd

#############################
# Images for Vanilla plugin #
#############################

if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then
  export JAVA_DOWNLOAD_URL=${JAVA_DOWNLOAD_URL:-"http://download.oracle.com/otn-pub/java/jdk/7u51-b13/jdk-7u51-linux-x64.tar.gz"}
  export OOZIE_HADOOP_V1_DOWNLOAD_URL=${OOZIE_HADOOP_V1_DOWNLOAD_URL:-"http://sahara-files.mirantis.com/oozie-4.0.0.tar.gz"}
  export OOZIE_HADOOP_V2_DOWNLOAD_URL=${OOZIE_HADOOP_V2_DOWNLOAD_URL:-"http://sahara-files.mirantis.com/oozie-4.0.0-hadoop-2.3.0.tar.gz"}
  export EXTJS_DOWNLOAD_URL=${EXTJS_DOWNLOAD_URL:-"http://extjs.com/deploy/ext-2.2.zip"}
  export HIVE_VERSION=${HIVE_VERSION:-"0.11.0"}

  ubuntu_elements_sequence="base vm ubuntu hadoop oozie mysql hive"
  fedora_elements_sequence="base vm fedora hadoop oozie mysql hive disable-firewall"
  centos_elements_sequence="vm rhel hadoop oozie mysql hive redhat-lsb disable-firewall"

  # Workaround for https://bugs.launchpad.net/diskimage-builder/+bug/1204824
  # https://bugs.launchpad.net/sahara/+bug/1252684
  if [ "$platform" = 'NAME="Ubuntu"' ]; then
    echo "**************************************************************"
    echo "WARNING: As a workaround for DIB bug 1204824, you are about to"
    echo "         create a Fedora and CentOS images that has SELinux    "
    echo "         disabled. Do not use these images in production.       "
    echo "**************************************************************"
    fedora_elements_sequence="$fedora_elements_sequence selinux-permissive"
    centos_elements_sequence="$centos_elements_sequence selinux-permissive"
    suffix=".selinux-permissive"
  fi

  if [ -n "$USE_MIRRORS" ]; then
    mirror_element=" apt-mirror"
    ubuntu_elements_sequence=$ubuntu_elements_sequence$mirror_element
    mirror_element=" yum-mirror"
    fedora_elements_sequence=$fedora_elements_sequence$mirror_element
  fi

{'fraction_non_alphanumeric': 0.1535641547861507, 'fraction_numerical': 0.011486761710794297, 'mean_word_length': 3.63595166163142, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3851615', 'n_tokens_mistral': 4542, 'n_tokens_neox': 4269, 'n_words': 1098}