<commit_before>#!/bin/bash -ex
##-------------------------------------------------------------------
## @copyright 2016 DennyZhang.com
## Licensed under MIT
##   https://raw.githubusercontent.com/DennyZhang/devops_public/tag_v1/LICENSE
##
## File : refresh_db_report.sh
## Author : DennyZhang.com <<EMAIL>>
## Description :
## --
## Created : <2015-09-24>
## Updated: Time-stamp: <2016-07-28 10:14:11>
##-------------------------------------------------------------------
################################################################################################
## env variables:
##       env_parameters:
##           export SSH_DOCKER_DAEMON="ssh root@172.17.0.1"
##           export KIBANA_DASHBOARD_URL="http://172.16.17.32:5601/#/dashboard/DataReport"
##           export DESTROY_CONTAINER=false
################################################################################################
. /etc/profile
[ -n "$DOWNLOAD_TAG_NAME" ] || export DOWNLOAD_TAG_NAME="tag_v2"
export DOWNLOAD_PREFIX="https://raw.githubusercontent.com/DennyZhang/devops_public/${DOWNLOAD_TAG_NAME}"
if [ ! -f /var/lib/devops/refresh_common_library.sh ]; then
    [ -d /var/lib/devops/ ] || (sudo mkdir -p  /var/lib/devops/ && sudo chmod 777 /var/lib/devops)
    wget -O /var/lib/devops/refresh_common_library.sh "$DOWNLOAD_PREFIX/common_library/refresh_common_library.sh"
fi
bash /var/lib/devops/refresh_common_library.sh "2593132520" "/var/lib/devops/devops_common_library.sh" \
     "${DOWNLOAD_PREFIX}/common_library/devops_common_library.sh"
. /var/lib/devops/devops_common_library.sh
################################################################################################
source_string "$env_parameters"

ensure_variable_isset "SSH_DOCKER_DAEMON is not given " "$SSH_DOCKER_DAEMON"
ensure_variable_isset "KIBANA_DASHBOARD_URL is not given " "$KIBANA_DASHBOARD_URL"
[ -n "$DESTROY_CONTAINER" ] || DESTROY_CONTAINER=false

echo "destroy old docker container of data-report"
$SSH_DOCKER_DAEMON docker stop data-report || true
$SSH_DOCKER_DAEMON docker rm data-report || true

if $DESTROY_CONTAINER; then
    echo "Quit after container destroy"
    exit 0
fi

echo "start elk data report container and services inside"
$SSH_DOCKER_DAEMON docker run -t -d --name data-report --privileged -p 5601:5601 denny/elk:datareport /usr/sbin/sshd -D

$SSH_DOCKER_DAEMON docker exec -t data-report "service logstash start"
$SSH_DOCKER_DAEMON docker exec -t data-report "service elasticsearch start"
$SSH_DOCKER_DAEMON docker exec -t data-report "service kibana4 start"

echo "Download and inject data file"
$SSH_DOCKER_DAEMON docker exec -t data-report "wget -O /tmp/db_summary_report.txt http://repo.fluigdata.com:18000/prodenv_db_summary_report/db_summary_report.txt"
$SSH_DOCKER_DAEMON docker exec -t data-report "cat /tmp/db_summary_report.txt >> /var/log/data_report.log"

echo "Check kibana dashboard"
$SSH_DOCKER_DAEMON docker exec -t data-report "lsof -i tcp:5601"

echo "Check DB Report: $KIBANA_DASHBOARD_URL"
## File : refresh_db_report.sh ends
<commit_msg>update data report<commit_after>#!/bin/bash -ex
##-------------------------------------------------------------------
## @copyright 2016 <EMAIL>
## Licensed under MIT
##   https://raw.githubusercontent.com/DennyZhang/devops_public/tag_v1/LICENSE
##
## File : refresh_db_report.sh
## Author : DennyZhang.com <<EMAIL>>
## Description :
## --
## Created : <2015-09-24>
## Updated: Time-stamp: <2016-07-28 14:01:05>
##-------------------------------------------------------------------
################################################################################################
## env variables:
##       env_parameters:
##           export SSH_DOCKER_DAEMON="ssh root@172.17.0.1"
##           export DATA_SOURCE_URL="http://XXX.XXX.XXX:18000/prodenv_db_summary_report/db_summary_report.txt"
##           export KIBANA_DASHBOARD_URL="http://172.16.17.32:5601/#/dashboard/DataReport"
##           export DESTROY_CONTAINER=false
################################################################################################
. /etc/profile
[ -n "$DOWNLOAD_TAG_NAME" ] || export DOWNLOAD_TAG_NAME="tag_v2"
export DOWNLOAD_PREFIX="https://raw.githubusercontent.com/DennyZhang/devops_public/${DOWNLOAD_TAG_NAME}"
if [ ! -f /var/lib/devops/refresh_common_library.sh ]; then
    [ -d /var/lib/devops/ ] || (sudo mkdir -p  /var/lib/devops/ && sudo chmod 777 /var/lib/devops)
    wget -O /var/lib/devops/refresh_common_library.sh "$DOWNLOAD_PREFIX/common_library/refresh_common_library.sh"
fi
bash /var/lib/devops/refresh_common_library.sh "2593132520" "/var/lib/devops/devops_common_library.sh" \
     "${DOWNLOAD_PREFIX}/common_library/devops_common_library.sh"
. /var/lib/devops/devops_common_library.sh
################################################################################################
source_string "$env_parameters"

ensure_variable_isset "SSH_DOCKER_DAEMON is not given " "$SSH_DOCKER_DAEMON"
ensure_variable_isset "KIBANA_DASHBOARD_URL is not given " "$KIBANA_DASHBOARD_URL"
[ -n "$DESTROY_CONTAINER" ] || DESTROY_CONTAINER=false

echo "destroy old docker container of data-report"
$SSH_DOCKER_DAEMON docker stop data-report || true
$SSH_DOCKER_DAEMON docker rm data-report || true

if $DESTROY_CONTAINER; then
    echo "Quit after container destroy"
    exit 0
fi

echo "start elk data report container"
$SSH_DOCKER_DAEMON docker run -t -d --name data-report --privileged -p 5601:5601 denny/elk:datareport /usr/sbin/sshd -D

echo "Download and inject data file"
$SSH_DOCKER_DAEMON docker exec -t data-report "wget -O /tmp/db_summary_report.txt $DATA_SOURCE_URL"
$SSH_DOCKER_DAEMON docker exec -t data-report "cat /tmp/db_summary_report.txt > /var/log/data_report.log"

echo "Start services inside docker container"
$SSH_DOCKER_DAEMON docker exec -t data-report "service logstash start"
$SSH_DOCKER_DAEMON docker exec -t data-report "service elasticsearch start"
$SSH_DOCKER_DAEMON docker exec -t data-report "service kibana4 start"

echo "Check kibana dashboard"
$SSH_DOCKER_DAEMON docker exec -t data-report "lsof -i tcp:5601"

echo "Check DB Report: $KIBANA_DASHBOARD_URL"
## File : refresh_db_report.sh ends

{'fraction_non_alphanumeric': 0.24227721170952612, 'fraction_numerical': 0.025877405790069547, 'mean_word_length': 4.1879194630872485, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 15, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '402595', 'n_tokens_mistral': 2204, 'n_tokens_neox': 2020, 'n_words': 401}