<commit_before>
#Load Env variables from File (maybe change to DB)
#using /home/ec2-user/Cloud1
#source /home/ec2-user/Cloud1
. /home/ec2-user/Cloud1
echo ""
echo "Loaded Config file"
echo ""
echo "$(tput setaf 2) Starting $VM_InstancesK Instances in AWS $(tput sgr 0)"
if [ $GCEKProvision -eq 1 ]; then
  echo "$(tput setaf 2) Starting $GCEVM_InstancesK Instances in GCE $(tput sgr 0)"
fi
echo "$(tput setaf 2) Starting $Container_InstancesK Container Instances $(tput sgr 0)"

echo ""
echo "$(tput setaf 2) Installing jq $(tput sgr 0)"
echo ""
wget http://stedolan.github.io/jq/download/linux64/jq

chmod +x ./jq

sudo cp -p jq /usr/bin

echo ""
echo "STARTING"
echo ""


echo ""
echo "$(tput setaf 2) Setting env variables for AWS CLI $(tput sgr 0)"
rm -rf ~/.aws/config
mkdir ~/.aws

touch ~/.aws/config

echo "[default]" > ~/.aws/config
echo "AWS_ACCESS_KEY_ID=$K1_AWS_ACCESS_KEY" >> ~/.aws/config
echo "AWS_SECRET_ACCESS_KEY=$K1_AWS_SECRET_KEY" >> ~/.aws/config
echo "AWS_DEFAULT_REGION=$K1_AWS_DEFAULT_REGION" >> ~/.aws/config

echo ""

#provision Consul via Docker machine or locally 
#depending on DynDDNS Usage variable ConsulDynDNSK
if [ $ConsulDynDNSK -eq 0 ]; then
  echo ""
  echo "$(tput setaf 2) Creating CONSUL VM via Docker Machine $(tput sgr 0)"
  echo ""
  #Create Docker Consul VM 
  docker-machine create --driver amazonec2 --amazonec2-access-key $K1_AWS_ACCESS_KEY --amazonec2-secret-key $K1_AWS_SECRET_KEY --amazonec2-vpc-id  $K1_AWS_VPC_ID --amazonec2-zone $K1_AWS_ZONE --amazonec2-region $K1_AWS_DEFAULT_REGION SPAWN-CONSUL

  #Opens Firewall Port for Consul
  aws ec2 authorize-security-group-ingress --group-name docker-machine --protocol tcp --port 8500 --cidr 0.0.0.0/0

  #Connects to remote VM

  docker-machine env SPAWN-CONSUL > /home/ec2-user/CONSUL1
  . /home/ec2-user/CONSUL1

  publicipCONSULK=$(docker-machine ip SPAWN-CONSUL)

  #Launches a remote Consul instance

  docker run -d -p 8400:8400 -p 8500:8500 -p 8600:53/udp -h node1 progrium/consul -server -bootstrap

else 
  echo ""
  echo "$(tput setaf 2) Creating a LOCAL CONSUL Container (DynDNS usage)  $(tput sgr 0)"
  echo ""
  
  #Launches a local Consul instance
  docker run -d --name ConsulDynDNS -p 8400:8400 -p 8500:8500 -p 8600:53/udp -h node1 progrium/consul -server -bootstrap

  #Stores local ip in a variable
  publicipCONSULK=$DynDNSK
  
fi



echo ----
echo "$(tput setaf 6) Consul RUNNING ON $publicipCONSULK:8500 $(tput sgr 0)"
echo publicipCONSULK=$publicipCONSULK
echo ----


echo ""
echo "$(tput setaf 2) Creating a LOCAL etcd instance  $(tput sgr 0)"
echo ""

docker run -d -v /usr/share/ca-certificates/:/etc/ssl/certs -p 4001:4001 -p 2380:2380 -p 2379:2379 --name etcdk quay.io/coreos/etcd -name etcd0 -advertise-client-urls http://${HostIP}:2379,http://${HostIP}:4001 -listen-client-urls http://0.0.0.0:2379,http://0.0.0.0:4001 -initial-advertise-peer-urls http://${HostIP}:2380 -listen-peer-urls http://0.0.0.0:2380 -initial-cluster-token <PASSWORD> -initial-cluster etcd0=http://${HostIP}:2380 -initial-cluster-state new

if [ $etcdbrowserprovision -eq 1 ]; then
  echo ""
  echo "$(tput setaf 2) Creating a etcd-browser instance in GCE $(tput sgr 0)"
  echo ""

  #Create Docker Receiver Instance in GCE
  #gcloud auth login
  gcloud auth activate-service-account $K2_GOOGLE_AUTH_EMAIL --key-file $GOOGLE_APPLICATION_CREDENTIALS --project $K2_GOOGLE_PROJECT

  docker-machine create -d google --google-project $K2_GOOGLE_PROJECT --google-machine-type g1-small etcd-browserk

  #
  #Open port for etcd-browser on GCE
  gcloud compute firewall-rules create etcd-browserk --allow tcp:8000 --source-ranges 0.0.0.0/0 --target-tags docker-machine --project $K2_GOOGLE_PROJECT

  #gcloud compute firewall-rules list docker-machine

  #Connects to remote VM

  docker-machine env etcd-browserk > /home/ec2-user/etcd-browserk
  . /home/ec2-user/etcd-browserk

  publicipetcdbrowser=$(docker-machine ip etcd-browserk)
  
  #launches etcd-browser containerized
  docker run -d --name etcd-browserk -p 0.0.0.0:8000:8000 --env ETCD_HOST=$DynDNSK kiodo/etcd-browser:latest
  
  #Register etcd-browser in etcd
  curl -L http://127.0.0.1:4001/v2/keys/etcd-browser/name -XPUT -d value=etcd-browserk
  curl -L http://127.0.0.1:4001/v2/keys/etcd-browser/ip -XPUT -d value=$publicipetcdbrowser
  curl -L http://127.0.0.1:4001/v2/keys/etcd-browser/port -XPUT -d value=8000
  echo ----
  echo "$(tput setaf 6) etcd-browser RUNNING ON $publicipetcdbrowser:8000 $(tput sgr 0)"
  echo "$(tput setaf 4) publicipetcdbrowser=$publicipetcdbrowser $(tput sgr 0)"
  echo ----
 fi
 
#Register Consul in etcd
if [ $ConsulDynDNSK -eq 0 ]; then
  curl -L http://127.0.0.1:4001/v2/keys/consul/name -XPUT -d value=SPAWN-CONSUL
else
  curl -L http://127.0.0.1:4001/v2/keys/consul/ip -XPUT -d value=$DynDNSK
fi
curl -L http://127.0.0.1:4001/v2/keys/consul/ip -XPUT -d value=$publicipCONSULK
curl -L http://127.0.0.1:4001/v2/keys/consul/port -XPUT -d value=8500

#register local ip and dns name in etcd
myipK=$( dig +short $DynDNSK @8.8.8.8)
curl -L http://127.0.0.1:4001/v2/keys/maininstance/ip -XPUT -d value=$myipK
fqnK=$(nslookup $myipK)
fqnK=${fqnK##*name = }
fqnK=${fqnK%.*}
#echo $fqn
curl -L http://127.0.0.1:4001/v2/keys/maininstance/name -XPUT -d value=$fqnK

#register local ip and dns name in Consul
curl -X PUT -d $fqnK http://$publicipCONSULK:8500/v1/kv/tc/maininstance/name
curl -X PUT -d $myipK http://$publicipCONSULK:8500/v1/kv/tc/maininstance/ip
 
#Provisions Receiver instance in GCE or AWS
if [ $GCEKProvision -eq 1 ]; then

  echo ""
  echo "$(tput setaf 2) Launching a Receiver Instance in GCE $(tput sgr 0)"
  echo ""


  #Create Docker Receiver Instance in GCE
  #gcloud auth login
  gcloud auth activate-service-account $K2_GOOGLE_AUTH_EMAIL --key-file $GOOGLE_APPLICATION_CREDENTIALS --project $K2_GOOGLE_PROJECT

  docker-machine create -d google --google-project $K2_GOOGLE_PROJECT --google-machine-type g1-small spawn-receiver

  #
  #Open port for Receiver on GCE
  gcloud compute firewall-rules create receiver-machines --allow tcp:$ReceiverPortK --source-ranges 0.0.0.0/0 --target-tags docker-machine --project $K2_GOOGLE_PROJECT

  #gcloud compute firewall-rules list docker-machine
<commit_msg>Update SpawnSwarmtcK.sh<commit_after>
#Load Env variables from File (maybe change to DB)
#using /home/ec2-user/Cloud1
#source /home/ec2-user/Cloud1
. /home/ec2-user/Cloud1
echo ""
echo "Loaded Config file"
echo ""
echo "$(tput setaf 2) Starting $VM_InstancesK Instances in AWS $(tput sgr 0)"
if [ $GCEKProvision -eq 1 ]; then
  echo "$(tput setaf 2) Starting $GCEVM_InstancesK Instances in GCE $(tput sgr 0)"
fi
echo "$(tput setaf 2) Starting $Container_InstancesK Container Instances $(tput sgr 0)"

echo ""
echo "$(tput setaf 2) Installing jq $(tput sgr 0)"
echo ""
wget http://stedolan.github.io/jq/download/linux64/jq

chmod +x ./jq

sudo cp -p jq /usr/bin

echo ""
echo "STARTING"
echo ""


echo ""
echo "$(tput setaf 2) Setting env variables for AWS CLI $(tput sgr 0)"
rm -rf ~/.aws/config
mkdir ~/.aws

touch ~/.aws/config

echo "[default]" > ~/.aws/config
echo "AWS_ACCESS_KEY_ID=$K1_AWS_ACCESS_KEY" >> ~/.aws/config
echo "AWS_SECRET_ACCESS_KEY=$K1_AWS_SECRET_KEY" >> ~/.aws/config
echo "AWS_DEFAULT_REGION=$K1_AWS_DEFAULT_REGION" >> ~/.aws/config

echo ""

#provision Consul via Docker machine or locally 
#depending on DynDDNS Usage variable ConsulDynDNSK
if [ $ConsulDynDNSK -eq 0 ]; then
  echo ""
  echo "$(tput setaf 2) Creating CONSUL VM via Docker Machine $(tput sgr 0)"
  echo ""
  #Create Docker Consul VM 
  docker-machine create --driver amazonec2 --amazonec2-access-key $K1_AWS_ACCESS_KEY --amazonec2-secret-key $K1_AWS_SECRET_KEY --amazonec2-vpc-id  $K1_AWS_VPC_ID --amazonec2-zone $K1_AWS_ZONE --amazonec2-region $K1_AWS_DEFAULT_REGION SPAWN-CONSUL

  #Opens Firewall Port for Consul
  aws ec2 authorize-security-group-ingress --group-name docker-machine --protocol tcp --port 8500 --cidr 0.0.0.0/0

  #Connects to remote VM

  docker-machine env SPAWN-CONSUL > /home/ec2-user/CONSUL1
  . /home/ec2-user/CONSUL1

  publicipCONSULK=$(docker-machine ip SPAWN-CONSUL)

  #Launches a remote Consul instance

  docker run -d -p 8400:8400 -p 8500:8500 -p 8600:53/udp -h node1 progrium/consul -server -bootstrap

else 
  echo ""
  echo "$(tput setaf 2) Creating a LOCAL CONSUL Container (DynDNS usage)  $(tput sgr 0)"
  echo ""
  
  #Launches a local Consul instance
  docker run -d --name ConsulDynDNS -p 8400:8400 -p 8500:8500 -p 8600:53/udp -h node1 progrium/consul -server -bootstrap

  #Stores local ip in a variable
  publicipCONSULK=$DynDNSK
  
fi



echo ----
echo "$(tput setaf 6) Consul RUNNING ON $publicipCONSULK:8500 $(tput sgr 0)"
echo publicipCONSULK=$publicipCONSULK
echo ----


echo ""
echo "$(tput setaf 2) Creating a LOCAL etcd instance  $(tput sgr 0)"
echo ""

docker run -d -v /usr/share/ca-certificates/:/etc/ssl/certs -p 4001:4001 -p 2380:2380 -p 2379:2379 --name etcdk quay.io/coreos/etcd -name etcd0 -advertise-client-urls http://${HostIP}:2379,http://${HostIP}:4001 -listen-client-urls http://0.0.0.0:2379,http://0.0.0.0:4001 -initial-advertise-peer-urls http://${HostIP}:2380 -listen-peer-urls http://0.0.0.0:2380 -initial-cluster-token etcd-cluster-1 -initial-cluster etcd0=http://${HostIP}:2380 -initial-cluster-state new

if [ $etcdbrowserprovision -eq 1 ]; then
  echo ""
  echo "$(tput setaf 2) Creating a etcd-browser instance in GCE $(tput sgr 0)"
  echo ""

  #Create Docker Receiver Instance in GCE
  #gcloud auth login
  gcloud auth activate-service-account $K2_GOOGLE_AUTH_EMAIL --key-file $GOOGLE_APPLICATION_CREDENTIALS --project $K2_GOOGLE_PROJECT

  docker-machine create -d google --google-project $K2_GOOGLE_PROJECT --google-machine-type g1-small etcd-browserk

  #
  #Open port for etcd-browser on GCE
  gcloud compute firewall-rules create etcd-browserk --allow tcp:8000 --source-ranges 0.0.0.0/0 --target-tags docker-machine --project $K2_GOOGLE_PROJECT

  #gcloud compute firewall-rules list docker-machine

  #Connects to remote VM

  docker-machine env etcd-browserk > /home/ec2-user/etcd-browserk
  . /home/ec2-user/etcd-browserk

  publicipetcdbrowser=$(docker-machine ip etcd-browserk)
  
  #launches etcd-browser containerized
  docker run -d --name etcd-browserk -p 0.0.0.0:8000:8000 --env ETCD_HOST=$DynDNSK kiodo/etcd-browser:latest
  
  #Register etcd-browser in etcd
  curl -L http://127.0.0.1:4001/v2/keys/etcd-browser/name -XPUT -d value=etcd-browserk
  curl -L http://127.0.0.1:4001/v2/keys/etcd-browser/ip -XPUT -d value=$publicipetcdbrowser
  curl -L http://127.0.0.1:4001/v2/keys/etcd-browser/port -XPUT -d value=8000
  echo ----
  echo "$(tput setaf 6) etcd-browser RUNNING ON $publicipetcdbrowser:8000 $(tput sgr 0)"
  echo "$(tput setaf 4) publicipetcdbrowser=$publicipetcdbrowser $(tput sgr 0)"
  echo ----
 fi
 
#Register Consul in etcd
if [ $ConsulDynDNSK -eq 0 ]; then
  curl -L http://127.0.0.1:4001/v2/keys/consul/name -XPUT -d value=SPAWN-CONSUL
else
  curl -L http://127.0.0.1:4001/v2/keys/consul/ip -XPUT -d value=$DynDNSK
fi
curl -L http://127.0.0.1:4001/v2/keys/consul/ip -XPUT -d value=$publicipCONSULK
curl -L http://127.0.0.1:4001/v2/keys/consul/port -XPUT -d value=8500

#register local ip and dns name in etcd
myipK=$( dig +short $DynDNSK @8.8.8.8)
curl -L http://127.0.0.1:4001/v2/keys/maininstance/ip -XPUT -d value=$myipK
fqnK=$(nslookup $myipK)
fqnK=${fqnK##*name = }
fqnK=${fqnK%.*}
#echo $fqn
curl -L http://127.0.0.1:4001/v2/keys/maininstance/name -XPUT -d value=$fqnK
if [ $ConsulDynDNSK -eq 1 ]; then
  curl -L http://127.0.0.1:4001/v2/keys/maininstance/dyndns -XPUT -d value=$DynDNSK
fi

#register local ip and dns name in Consul
curl -X PUT -d $fqnK http://$publicipCONSULK:8500/v1/kv/tc/maininstance/name
curl -X PUT -d $myipK http://$publicipCONSULK:8500/v1/kv/tc/maininstance/ip
if [ $ConsulDynDNSK -eq 1 ]; then
  curl -X PUT -d $DynDNSK http://$publicipCONSULK:8500/v1/kv/tc/maininstance/dyndns
fi
 
#Provisions Receiver instance in GCE or AWS
if [ $GCEKProvision -eq 1 ]; then

  echo ""
  echo "$(tput setaf 2) Launching a Receiver Instance in GCE $(tput sgr 0)"
  echo ""


  #Create Docker Receiver Instance in GCE
  #gcloud auth login
  gcloud auth activate-service-account $K2_GOOGLE_AUTH_EMAIL --key-file $GOOGLE_APPLICATION_CREDENTIALS --project $K2_GOOGLE_PROJECT

  docker-machine create -d google --google-project $K2_GOOGLE_PROJECT --google-machine-type g1-small spawn-receiver

  #
  #Open port for Receiver on GCE
  gcloud compute firewall-rules create receiver-machines --allow tcp:$ReceiverPortK --source-ranges 0.0.0.0/0 --target-tags docker-machine --project $K2_GOOGLE_PROJECT

  #gcloud compute firewall-rules list docker-machine

{'fraction_non_alphanumeric': 0.13411170716314294, 'fraction_numerical': 0.05609024928999685, 'mean_word_length': 3.7765636774679727, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 35, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3168139', 'n_tokens_mistral': 5483, 'n_tokens_neox': 4972, 'n_words': 1279}