<commit_before>#!/bin/bash

#
# Discover the APP_BASE from the location of this script.
#
if [ -z "$APP_BASE" ] ; then
  DIRNAME=`dirname "$0"`
  APP_BASE=`cd "$DIRNAME"; pwd`
  export APP_BASE
fi

OPENSHIFT_IMAGE=openshift/origin:latest
OPENSHIFT_ROUTER_IMAGE=openshift/origin-haproxy-router:latest
REGISTRY_IMAGE=registry:latest
CADVISOR_IMAGE=google/cadvisor:0.8.0
INFLUXDB_IMAGE=tutum/influxdb:latest
FABRIC8_CONSOLE_IMAGE=fabric8/hawtio-kubernetes:latest
KIBANA_IMAGE=jimmidyson/kibana4:latest
ELASTICSEARCH_IMAGE=dockerfile/elasticsearch:latest
LOGSPOUT_IMAGE=jimmidyson/logspout-kube:latest
GRAFANA_IMAGE=jimmidyson/grafana:latest

MINIMUM_IMAGES="${OPENSHIFT_IMAGE} ${FABRIC8_CONSOLE_IMAGE} ${REGISTRY_IMAGE}"
ALL_IMAGES="${MINIMUM_IMAGES} ${OPENSHIFT_ROUTER_IMAGE} ${CADVISOR_IMAGE} ${INFLUXDB_IMAGE} ${KIBANA_IMAGE} ${ELASTICSEARCH_IMAGE} ${LOGSPOUT_IMAGE} ${GRAFANA_IMAGE}"
DEPLOY_IMAGES="${MINIMUM_IMAGES}"
UPDATE_IMAGES=0
DEPLOY_ALL=0
CLEANUP=0
DONT_RUN=0
FABRIC8_VAGRANT_IP=172.28.128.4

while getopts "fud:kp" opt; do
  case $opt in
    k)
      DEPLOY_IMAGES="${ALL_IMAGES}"
      DEPLOY_ALL=1
      ;;
    f)
      echo "Cleaning up all existing k8s containers"
      docker rm -fv openshift cadvisor || true
      RUNNING_CONTAINERS=`docker ps -a | grep k8s | cut -c 1-12`
      test -z "$RUNNING_CONTAINERS" || docker rm -fv $RUNNING_CONTAINERS
      CLEANUP=1
      echo
      ;;
    u)
      UPDATE_IMAGES=1
      ;;
    p)
      UPDATE_IMAGES=1
      DONT_RUN=1
      ;;
    d)
      DOCKER_IP=$OPTARG
      ;;
    \?)
      echo "Invalid option: -$OPTARG" >&2
      echo
      ;;
  esac
done

echo "Validating your environment..."
echo

if [ ${UPDATE_IMAGES} -eq 1 ]; then
  echo "Updating all necessary images"
  for image in ${DEPLOY_IMAGES}; do
    docker pull $image
  done
  echo
fi

for image in ${DEPLOY_IMAGES}; do
  (
    IFS=':' read -a splitimage <<< "$image"
    docker images | grep -qEo "${splitimage[0]}\W+${splitimage[1]}" || (echo "Missing necessary Docker image: $image" && docker pull $image && echo)
  )
done

if [ ${CLEANUP} -eq 1 ]; then
  docker run --rm --privileged -v /var/lib:/var/lib --entrypoint=rm ${OPENSHIFT_IMAGE} -rf /var/lib/openshift/
fi

if [ ${DONT_RUN} -eq 1 ]; then
  echo "Terminating before running any containers"
  exit 0
fi

echo "Validating firewall rules"
RULE="INPUT -d 172.17.42.1 -s 172.17.0.0/16 -j ACCEPT"
RULE_OUTPUT=$( { docker run --rm --privileged --net=host --entrypoint=iptables ${OPENSHIFT_IMAGE} -C $RULE; } 2>&1 )
test -n "$RULE_OUTPUT" && docker run --rm --privileged --net=host --entrypoint=iptables ${OPENSHIFT_IMAGE} -I $RULE
RULE="INPUT -d 172.17.0.0/16 -s 172.16.58.3/16 -j ACCEPT"
RULE_OUTPUT=$( { docker run --rm --privileged --net=host --entrypoint=iptables ${OPENSHIFT_IMAGE} -C $RULE; } 2>&1 )
test -n "$RULE_OUTPUT" && docker run --rm --privileged --net=host --entrypoint=iptables ${OPENSHIFT_IMAGE} -I $RULE
RULE="INPUT -d 172.16.58.3/16 -s 172.17.0.0/16 -j ACCEPT"
RULE_OUTPUT=$( { docker run --rm --privileged --net=host --entrypoint=iptables ${OPENSHIFT_IMAGE} -C $RULE; } 2>&1 )
test -n "$RULE_OUTPUT" && docker run --rm --privileged --net=host --entrypoint=iptables ${OPENSHIFT_IMAGE} -I $RULE
RULE="INPUT -d 172.30.17.0/24 -s 172.17.0.0/16 -j ACCEPT"
RULE_OUTPUT=$( { docker run --rm --privileged --net=host --entrypoint=iptables ${OPENSHIFT_IMAGE} -C $RULE; } 2>&1 )
test -n "$RULE_OUTPUT" && docker run --rm --privileged --net=host --entrypoint=iptables ${OPENSHIFT_IMAGE} -I $RULE
RULE="INPUT -d 172.17.0.0/16 -s 172.30.17.0/24 -j ACCEPT"
RULE_OUTPUT=$( { docker run --rm --privileged --net=host --entrypoint=iptables ${OPENSHIFT_IMAGE} -C $RULE; } 2>&1 )
test -n "$RULE_OUTPUT" && docker run --rm --privileged --net=host --entrypoint=iptables ${OPENSHIFT_IMAGE} -I $RULE
echo

# TODO it would be nice if we could tell easily if these routes have already been applied so we don't have to do this each time
if [[ $OSTYPE == darwin* ]]; then
    if [ -z "$DOCKER_IP" ] ; then
      export DOCKER_IP=`boot2docker ip 2> /dev/null`
    fi

    echo "Adding network routes to 172.17.0.0/24, 172.30.17.0/24 & 172.16.17.32/24 via $DOCKER_IP so that the host operating system can see pods and services inside OpenShift"
    sudo route -n add 172.17.0.0/24 $DOCKER_IP
    sudo route -n add 172.30.17.0/24 $DOCKER_IP
    sudo route -n add 172.16.17.32/24 $DOCKER_IP
fi

export DOCKER_IP=${DOCKER_IP:-127.0.0.1}
export KUBERNETES=http://$DOCKER_IP:8443

# using an env var but ideally we'd use an alias ;)
KUBE="docker run --rm -i --net=host ${OPENSHIFT_IMAGE} cli --insecure-skip-tls-verify=true"

OPENSHIFT_CONTAINER=$(docker run -d --name=openshift -v /var/run/docker.sock:/var/run/docker.sock -v /var/lib/openshift:/var/lib/openshift --privileged --net=host ${OPENSHIFT_IMAGE} start --portal-net='172.30.17.0/24' --cors-allowed-origins='.*')

validateService()
{
  echo "Waiting for $1"
  while true; do
    curl -k -s -o /dev/null --connect-timeout 1 $2 && break || sleep 1
  done
}

validateService "Kubernetes master" $KUBERNETES

if [ ${DEPLOY_ALL} -eq 1 ]; then
  # Have to run it privileged otherwise not working on CentOS7
  CADVISOR_CONTAINER=$(docker run -d --name=cadvisor --privileged -p 4194:8080 \
    --volume=/:/rootfs:ro \
    --volume=/var/run:/var/run:rw \
    --volume=/sys:/sys:ro \
    --volume=/var/lib/docker/:/var/lib/docker:ro \
    ${CADVISOR_IMAGE})
fi

if [ -f "$APP_BASE/fabric8.json" ]; then
  cat $APP_BASE/fabric8.json | $KUBE apply -f -
  cat $APP_BASE/registry.json | $KUBE apply -f -
  if [ ${DEPLOY_ALL} -eq 1 ]; then
    cat $APP_BASE/influxdb.json | $KUBE apply -f -
    cat $APP_BASE/elasticsearch.json | $KUBE apply -f -
    cat $APP_BASE/logspout.yml | $KUBE apply -f -
    cat $APP_BASE/kibana.yml | $KUBE apply -f -
    cat $APP_BASE/grafana.yml | $KUBE apply -f -
    cat $APP_BASE/router.json | $KUBE create  -f -
  fi
else
  curl -s https://raw.githubusercontent.com/fabric8io/fabric8/master/bin/fabric8.json | $KUBE apply -f -
  curl -s https://raw.githubusercontent.com/fabric8io/fabric8/master/bin/registry.json | $KUBE apply -f -
  if [ ${DEPLOY_ALL} -eq 1 ]; then
    curl -s https://raw.githubusercontent.com/fabric8io/fabric8/master/bin/influxdb.json | $KUBE apply -f -
    curl -s https://raw.githubusercontent.com/fabric8io/fabric8/master/bin/elasticsearch.json | $KUBE apply -f -
    curl -s https://raw.githubusercontent.com/fabric8io/fabric8/master/bin/logspout.yml | $KUBE apply -f -
    curl -s https://raw.githubusercontent.com/fabric8io/fabric8/master/bin/kibana.yml | $KUBE apply -f -
    curl -s https://raw.githubusercontent.com/fabric8io/fabric8/master/bin/grafana.yml | $KUBE apply -f -
    curl -s https://raw.githubusercontent.com/fabric8io/fabric8/master/bin/router.json | $KUBE create -f -
  fi
fi<commit_msg>enable the app library service in the startup script<commit_after>#!/bin/bash

#
# Discover the APP_BASE from the location of this script.
#
if [ -z "$APP_BASE" ] ; then
  DIRNAME=`dirname "$0"`
  APP_BASE=`cd "$DIRNAME"; pwd`
  export APP_BASE
fi

FABRIC8_VERSION=2.0.29
OPENSHIFT_IMAGE=openshift/origin:latest
OPENSHIFT_ROUTER_IMAGE=openshift/origin-haproxy-router:latest
REGISTRY_IMAGE=registry:latest
CADVISOR_IMAGE=google/cadvisor:0.8.0
INFLUXDB_IMAGE=tutum/influxdb:latest
FABRIC8_CONSOLE_IMAGE=fabric8/hawtio-kubernetes:latest
KIBANA_IMAGE=jimmidyson/kibana4:latest
ELASTICSEARCH_IMAGE=dockerfile/elasticsearch:latest
LOGSPOUT_IMAGE=jimmidyson/logspout-kube:latest
GRAFANA_IMAGE=jimmidyson/grafana:latest
APP_LIBRARY_IMAGE=fabric8/app-library:${FABRIC8_VERSION}

MINIMUM_IMAGES="${OPENSHIFT_IMAGE} ${FABRIC8_CONSOLE_IMAGE} ${APP_LIBRARY_IMAGE} ${REGISTRY_IMAGE}"
ALL_IMAGES="${MINIMUM_IMAGES} ${OPENSHIFT_ROUTER_IMAGE} ${CADVISOR_IMAGE} ${INFLUXDB_IMAGE} ${KIBANA_IMAGE} ${ELASTICSEARCH_IMAGE} ${LOGSPOUT_IMAGE} ${GRAFANA_IMAGE}"
DEPLOY_IMAGES="${MINIMUM_IMAGES}"
UPDATE_IMAGES=0
DEPLOY_ALL=0
CLEANUP=0
DONT_RUN=0
FABRIC8_VAGRANT_IP=172.28.128.4

while getopts "fud:kp" opt; do
  case $opt in
    k)
      DEPLOY_IMAGES="${ALL_IMAGES}"
      DEPLOY_ALL=1
      ;;
    f)
      echo "Cleaning up all existing k8s containers"
      docker rm -fv openshift cadvisor || true
      RUNNING_CONTAINERS=`docker ps -a | grep k8s | cut -c 1-12`
      test -z "$RUNNING_CONTAINERS" || docker rm -fv $RUNNING_CONTAINERS
      CLEANUP=1
      echo
      ;;
    u)
      UPDATE_IMAGES=1
      ;;
    p)
      UPDATE_IMAGES=1
      DONT_RUN=1
      ;;
    d)
      DOCKER_IP=$OPTARG
      ;;
    \?)
      echo "Invalid option: -$OPTARG" >&2
      echo
      ;;
  esac
done

echo "Validating your environment..."
echo

if [ ${UPDATE_IMAGES} -eq 1 ]; then
  echo "Updating all necessary images"
  for image in ${DEPLOY_IMAGES}; do
    docker pull $image
  done
  echo
fi

for image in ${DEPLOY_IMAGES}; do
  (
    IFS=':' read -a splitimage <<< "$image"
    docker images | grep -qEo "${splitimage[0]}\W+${splitimage[1]}" || (echo "Missing necessary Docker image: $image" && docker pull $image && echo)
  )
done

if [ ${CLEANUP} -eq 1 ]; then
  docker run --rm --privileged -v /var/lib:/var/lib --entrypoint=rm ${OPENSHIFT_IMAGE} -rf /var/lib/openshift/
fi

if [ ${DONT_RUN} -eq 1 ]; then
  echo "Terminating before running any containers"
  exit 0
fi

echo "Validating firewall rules"
RULE="INPUT -d 172.17.42.1 -s 172.17.0.0/16 -j ACCEPT"
RULE_OUTPUT=$( { docker run --rm --privileged --net=host --entrypoint=iptables ${OPENSHIFT_IMAGE} -C $RULE; } 2>&1 )
test -n "$RULE_OUTPUT" && docker run --rm --privileged --net=host --entrypoint=iptables ${OPENSHIFT_IMAGE} -I $RULE
RULE="INPUT -d 172.17.0.0/16 -s 172.16.58.3/16 -j ACCEPT"
RULE_OUTPUT=$( { docker run --rm --privileged --net=host --entrypoint=iptables ${OPENSHIFT_IMAGE} -C $RULE; } 2>&1 )
test -n "$RULE_OUTPUT" && docker run --rm --privileged --net=host --entrypoint=iptables ${OPENSHIFT_IMAGE} -I $RULE
RULE="INPUT -d 172.16.58.3/16 -s 172.17.0.0/16 -j ACCEPT"
RULE_OUTPUT=$( { docker run --rm --privileged --net=host --entrypoint=iptables ${OPENSHIFT_IMAGE} -C $RULE; } 2>&1 )
test -n "$RULE_OUTPUT" && docker run --rm --privileged --net=host --entrypoint=iptables ${OPENSHIFT_IMAGE} -I $RULE
RULE="INPUT -d 172.30.17.0/24 -s 172.17.0.0/16 -j ACCEPT"
RULE_OUTPUT=$( { docker run --rm --privileged --net=host --entrypoint=iptables ${OPENSHIFT_IMAGE} -C $RULE; } 2>&1 )
test -n "$RULE_OUTPUT" && docker run --rm --privileged --net=host --entrypoint=iptables ${OPENSHIFT_IMAGE} -I $RULE
RULE="INPUT -d 172.17.0.0/16 -s 172.30.17.0/24 -j ACCEPT"
RULE_OUTPUT=$( { docker run --rm --privileged --net=host --entrypoint=iptables ${OPENSHIFT_IMAGE} -C $RULE; } 2>&1 )
test -n "$RULE_OUTPUT" && docker run --rm --privileged --net=host --entrypoint=iptables ${OPENSHIFT_IMAGE} -I $RULE
echo

# TODO it would be nice if we could tell easily if these routes have already been applied so we don't have to do this each time
if [[ $OSTYPE == darwin* ]]; then
    if [ -z "$DOCKER_IP" ] ; then
      export DOCKER_IP=`boot2docker ip 2> /dev/null`
    fi

    echo "Adding network routes to 172.17.0.0/24, 172.30.17.0/24 & 172.16.17.32/24 via $DOCKER_IP so that the host operating system can see pods and services inside OpenShift"
    sudo route -n add 172.17.0.0/24 $DOCKER_IP
    sudo route -n add 172.30.17.0/24 $DOCKER_IP
    sudo route -n add 172.16.17.32/24 $DOCKER_IP
fi

export DOCKER_IP=${DOCKER_IP:-127.0.0.1}
export KUBERNETES=http://$DOCKER_IP:8443

# using an env var but ideally we'd use an alias ;)
KUBE="docker run --rm -i --net=host ${OPENSHIFT_IMAGE} cli --insecure-skip-tls-verify=true"

OPENSHIFT_CONTAINER=$(docker run -d --name=openshift -v /var/run/docker.sock:/var/run/docker.sock -v /var/lib/openshift:/var/lib/openshift --privileged --net=host ${OPENSHIFT_IMAGE} start --portal-net='172.30.17.0/24' --cors-allowed-origins='.*')

validateService()
{
  echo "Waiting for $1"
  while true; do
    curl -k -s -o /dev/null --connect-timeout 1 $2 && break || sleep 1
  done
}

validateService "Kubernetes master" $KUBERNETES

if [ ${DEPLOY_ALL} -eq 1 ]; then
  # Have to run it privileged otherwise not working on CentOS7
  CADVISOR_CONTAINER=$(docker run -d --name=cadvisor --privileged -p 4194:8080 \
    --volume=/:/rootfs:ro \
    --volume=/var/run:/var/run:rw \
    --volume=/sys:/sys:ro \
    --volume=/var/lib/docker/:/var/lib/docker:ro \
    ${CADVISOR_IMAGE})
fi

if [ -f "$APP_BASE/fabric8.json" ]; then
  cat $APP_BASE/fabric8.json | $KUBE apply -f -
  $KUBE apply -f  http://central.maven.org/maven2/io/fabric8/jube/images/fabric8/app-library/${FABRIC8_VERSION}/app-library-${FABRIC8_VERSION}-kubernetes.json
  cat $APP_BASE/registry.json | $KUBE apply -f -

  if [ ${DEPLOY_ALL} -eq 1 ]; then
    cat $APP_BASE/influxdb.json | $KUBE apply -f -
    cat $APP_BASE/elasticsearch.json | $KUBE apply -f -
    cat $APP_BASE/logspout.yml | $KUBE apply -f -
    cat $APP_BASE/kibana.yml | $KUBE apply -f -
    cat $APP_BASE/grafana.yml | $KUBE apply -f -
    cat $APP_BASE/router.json | $KUBE create  -f -
  fi
else
  curl -s https://raw.githubusercontent.com/fabric8io/fabric8/master/bin/fabric8.json | $KUBE apply -f -
  $KUBE apply -f  http://central.maven.org/maven2/io/fabric8/jube/images/fabric8/app-library/${FABRIC8_VERSION}/app-library-${FABRIC8_VERSION}-kubernetes.json
  curl -s https://raw.githubusercontent.com/fabric8io/fabric8/master/bin/registry.json | $KUBE apply -f -

  if [ ${DEPLOY_ALL} -eq 1 ]; then
    curl -s https://raw.githubusercontent.com/fabric8io/fabric8/master/bin/influxdb.json | $KUBE apply -f -
    curl -s https://raw.githubusercontent.com/fabric8io/fabric8/master/bin/elasticsearch.json | $KUBE apply -f -
    curl -s https://raw.githubusercontent.com/fabric8io/fabric8/master/bin/logspout.yml | $KUBE apply -f -
    curl -s https://raw.githubusercontent.com/fabric8io/fabric8/master/bin/kibana.yml | $KUBE apply -f -
    curl -s https://raw.githubusercontent.com/fabric8io/fabric8/master/bin/grafana.yml | $KUBE apply -f -
    curl -s https://raw.githubusercontent.com/fabric8io/fabric8/master/bin/router.json | $KUBE create -f -
  fi
fi
{'fraction_non_alphanumeric': 0.1511843607305936, 'fraction_numerical': 0.03817066210045662, 'mean_word_length': 3.48113810741688, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 17, 'https://': 16, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 38, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1050991', 'n_tokens_mistral': 5927, 'n_tokens_neox': 5394, 'n_words': 1256}